diff --git a/firmware/Makefile b/firmware/Makefile new file mode 100755 index 0000000..f40e015 --- /dev/null +++ b/firmware/Makefile @@ -0,0 +1,28 @@ +export TARGET_DIRS = $(PWD)/targets/RoCEv2_1GbeRudpKcu105Examples \ + $(PWD)/targets/RoCEv2_10GbeRudpKcu105Example \ + $(PWD)/targets/RoCEv2_Rj45RudpKcu105Example \ + $(PWD)/targets/Simple1GbeRudpKcu105Example \ + $(PWD)/targets/Simple10GbeRudpKcu105Example \ + $(PWD)/targets/SimpleRj45RudpKcu105Example + +.PHONY: all build clean + +# Default +all: build + +# Check variables +test: + @echo TARGET_DIRS: + @echo -e "$(foreach ARG,$(TARGET_DIRS),\t$(ARG)\n)" + +# Clean all firmware builds +clean: + for i in $(TARGET_DIRS); do \ + cd $$i; make clean; \ + done + +# Build targets +build: + for i in $(TARGET_DIRS); do \ + echo $$i; cd $$i; make clean; rm -rf images; make; \ + done diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py new file mode 100644 index 0000000..7e0a471 --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py @@ -0,0 +1,33 @@ +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- + +import surf.protocols.ssi as ssi +import surf.axi as axi + +import pyrogue as pr +import simple_10gbe_rudp_kcu105_example as baseBoard + +# class App(baseBoard.App): +class App(pr.Device): + def __init__( self, **kwargs): + super().__init__(**kwargs) + + self.add(ssi.SsiPrbsTx( + offset = 0x0002_0000, + clock_freq = 156.25e6, + expand = True, + )) + + self.add(axi.AxiStreamMonAxiL( + name = 'RdmaAxisMon', + offset = 0x0003_0000, + numberLanes = 1, + expand = True, + )) diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py new file mode 100644 index 0000000..5ffef20 --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py @@ -0,0 +1,235 @@ +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- + +import time + +import pyrogue as pr +import pyrogue.protocols +import pyrogue.utilities.fileio +import pyrogue.utilities.prbs +import pyrogue.interfaces.simulation + +import rogue +import rogue.hardware.axi +import rogue.interfaces.stream +import rogue.protocols.packetizer +import rogue.utilities.fileio + +import simple_10gbe_rudp_kcu105_example as baseBoard +import rocev2_10gbe_rudp_kcu105_example as roceBoard + +rogue.Version.minVersion('6.15.0') + +class Root(pr.Root): + def __init__(self, + rocev2Cfg, # RoCEv2ServerCfg (required): host-NIC config + ip = '192.168.2.10', # FPGA IP address (RUDP transport) + zmqSrvPort = 9099, # Set to zero if dynamic (instead of static) + useDcqcn = True, # Enable DCQCN congestion control in the Core + transportCfg = None, # RoCEv2TransportCfg: transport/QP-tuning knobs + p2p = True, # Default to switchless point-to-point bring-up (Not-ECT, DCQCN bypassed) + dscp = 26, # Managed-fabric only (p2p=False): IP-header DSCP (26 = AF31) + ecn = 2, # Managed-fabric only (p2p=False): IP-header ECN (2 = ECT(0) = b"10") + enableDcqcn = True, # Managed-fabric only (p2p=False): run FW DCQCN (False bypasses it) + **kwargs): + super().__init__(timeout=5.0, **kwargs) + + # Bring-up posture: p2p drives the egress ECN/DSCP and DCQCN-bypass setup + # in start(). The dscp/ecn/enableDcqcn knobs only apply when p2p is False. + self._p2p = p2p + self._dscp = dscp + self._ecn = ecn + self._enableDcqcn = enableDcqcn + + # Single transport/QP-tuning cfg forwarded into BOTH engine.setupConnection() + # and server.completeConnection() so the FPGA and host sides cannot drift. + self._transportCfg = transportCfg if transportCfg is not None else pr.protocols.RoCEv2TransportCfg() + + ################################################################# + + self.zmqServer = pyrogue.interfaces.ZmqServer(root=self, addr='127.0.0.1', port=zmqSrvPort) + self.addInterface(self.zmqServer) + + ################################################################# + + # UDP/RSSI client + self.rudp = [None for i in range(1)] + for i in range(1): + self.rudp[i] = pr.protocols.UdpRssiPack( + name = f'SwRudpClient[{i}]', + host = ip, + port = 8192 + i, + packVer = 2, + jumbo = (i > 0), + expand = False, + ) + self.add(self.rudp[i]) + + # SRPv3 for register access + self.srp = rogue.protocols.srp.SrpV3() + self.srp == self.rudp[0].application(0) + + ################################################################# + + # ---- RoCEv2 receive channel (additive, alongside RUDP) ---- + self.add(baseBoard.Core( + offset = 0x0000_0000, + memBase = self.srp, + rocev2 = True, + dcqcn = useDcqcn, + expand = False, + )) + + self.add(roceBoard.App( + offset = 0x8000_0000, + memBase = self.srp, + expand = True, + )) + + ################################################################# + + # RoCEv2 receive server (resolves cfg sentinels + auto-detects GID internally). + self._rdmaRx = self.add(pr.protocols.RoCEv2Server( + name = 'rdmaRx', + rocev2Cfg = rocev2Cfg, + expand = False, + )) + + # Host-side PRBS data-integrity check on the RDMA receive stream. + self.prbsRx = pr.utilities.prbs.PrbsRx( + name = 'PrbsRx', + width = 64, + checkPayload = True, + expand = True, + ) + self.add(self.prbsRx) + + # Strip the FW AxiStreamPacketizer2 framing before the PRBS check. + # CoreV2(ibCRC=False, obCRC=False, enSsi=True): no inbound CRC (FW + # CRC_MODE_G="NONE"), SSI SOF/EOF enabled. FW emits OUTPUT_TDEST_G=0, so the + # depacketized payload exits on application(0). + self._depack = rogue.protocols.packetizer.CoreV2(False, False, True) + self.rdmaRx.stream >> self._depack.transport() + self._depack.application(0) >> self.prbsRx + + ################################################################# + + def start(self, **kwargs): + super().start(**kwargs) + + # pr.Root.__enter__ calls start() directly and __exit__/stop() only runs if + # __enter__ returns, so an exception in the hand-off would skip teardown and + # leak the transport/poll threads or a half-established QP. Unwind through + # stop() before re-raising; teardownConnection() is best-effort with no live + # QP, and stop() SoftResets the engine to clear any partial QP/PSN state. + try: + self._handoff() + except Exception: + self.stop() + raise + + def _handoff(self): + # Host<->FPGA hand-off. super().start() already brought up the RUDP/SRP + # transport and the server's host-side _start(), so the metadata bus is + # reachable. Forward the single transportCfg into BOTH the engine and the + # server so the two sides stay in sync. + cfg = self._transportCfg + + params = self.rdmaRx.getHostParams() + fpga = self.Core.RoCEv2AxiStreamRdma.Engine.setupConnection( + **params._asdict(), + pmtu = cfg.pmtu, + minRnrTimer = cfg.minRnrTimer, + rnrRetry = cfg.rnrRetry, + retryCount = cfg.retryCount, + ) + self.rdmaRx.completeConnection( + fpga.fpgaQpn, + fpgaLkey = fpga.lkey, + pmtu = cfg.pmtu, + minRnrTimer = cfg.minRnrTimer, + rnrRetry = cfg.rnrRetry, + retryCount = cfg.retryCount, + ) + + # Point the FW UDP engine at the host NIC. Ordering vs the hand-off above + # is free: that runs over the SRP register bus (port 8192), while the 4791 + # RoCEv2 datapath only carries traffic once DispatchEnable is armed later. + hostIp = self.rdmaRx.HostIp.get() + self.Core.UdpEngine.ClientRemotePort[0].set(4791) + self.Core.UdpEngine.ClientRemoteIp[0].set(hostIp) + + # Configure egress ECN/DSCP and DCQCN posture for the deployment. + # + # p2p (default): switchless link. Force Not-ECT, no DSCP marking, and + # bypass DCQCN. The FW reset default is already Not-ECT/DSCP=0 (Rudp.vhd + # U_UDP); set it explicitly so the posture holds regardless of prior + # runtime state. ECT(0) would opt the flow into the host NIC's hardware + # DCQCN: with no ECN-marking fabric here, throttle-induced microbursts get + # CE-marked, the NIC returns CNPs, and the FW throttle self-sustains (CNPs + # reset the rate-increase timer faster than it fires) — throughput + # collapses until a source drain. Not-ECT removes that spurious trigger. + # + # Managed fabric (p2p=False): apply the configured DSCP/ECN to join the + # switch lossless/ECN traffic class, leaving DCQCN active unless + # enableDcqcn was cleared. + if self._p2p: + self.Core.UdpEngine.EcnFlag.set(0) # 0 = Not-ECT + self.Core.UdpEngine.Dscp.set(0) + else: + self.Core.UdpEngine.EcnFlag.set(self._ecn) + self.Core.UdpEngine.Dscp.set(self._dscp) + + # DcqcnBypass: bypass FW DCQCN for p2p, or for an explicit enableDcqcn=False on a + # fabric. RNR backoff is independent — set via transportCfg at setupConnection above. + self.setP2pMode(self._p2p or not self._enableDcqcn) + + self.rdmaRx.printConnInfo() + + + def setP2pMode(self, enable): + """Point-to-point bring-up toggle for FW DCQCN. + + Toggles the live AXI-Lite register Core.RoCEv2AxiStreamRdma.Dcqcn.DcqcnBypass + so the bypass takes effect immediately (True = gate CNP + clamp Rc/Rt to line + rate). + + RNR backoff is deliberately NOT set here: min_rnr_timer is host-NIC QP state + fixed at QP setup and is owned solely by self._transportCfg.minRnrTimer, which + start() forwards into setupConnection()/completeConnection(). Decoupling the RNR + timer from the DCQCN-bypass toggle lets a slow (softRoCE) responder keep a larger + backoff while still bypassing DCQCN on a switchless link — forcing the minimal + 0.01ms backoff here storms RNR-NAKs on a software responder. + """ + self.Core.RoCEv2AxiStreamRdma.Dcqcn.DcqcnBypass.set(enable) + + def stop(self) -> None: + """Tear down the FPGA QP before transport is stopped.""" + # Must run before super().stop(): pr.Root.stop() -> Device._stop() recurses in + # ADD order, and self.rudp[0] was added before Core, so the transport tears + # down first. An Engine._stop() hook would then fire after the metadata bus is + # already dead (register timeout — verified on hardware). So disarm the + # dispatcher and tear down the QP here, while the transport is still up. + # Guarded so a missing node (or any teardown error) never aborts stop() — + # super().stop() must always run to release the transport/poll threads. + try: + self.Core.RoCEv2AxiStreamRdma.Core.DispatchEnable.set(False) + time.sleep(0.1) # let the in-flight SEND drain before QP teardown + self.Core.RoCEv2AxiStreamRdma.Engine.teardownConnection() + # Force the FW RoceConfigurator back to IDLE. It has no response timeout, so + # an out-of-order / unexpected-state DESTROY can wedge it in GET_RESPONSE_S + # (the multi-second "DESTROY QP/MR/PD timeout"); a SoftReset pulse clears that + # and any stale QP/PSN state so the next bring-up starts clean. + self.Core.RoCEv2AxiStreamRdma.Engine.SoftReset() + except Exception as e: + print(f"Root.stop: RoCEv2 teardown skipped/failed ({e}); " + f"proceeding to transport stop.") + finally: + super().stop() diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py new file mode 100644 index 0000000..6d746cc --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py @@ -0,0 +1,2 @@ +from rocev2_10gbe_rudp_kcu105_example._App import * +from rocev2_10gbe_rudp_kcu105_example._Root import * diff --git a/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py b/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py index 8aa3f87..807f8d5 100644 --- a/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py +++ b/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py @@ -17,11 +17,14 @@ import surf.ethernet.ten_gig as mac import surf.protocols.rssi as rssi import surf.xilinx as xil +import surf.ethernet.roce as roce class Core(pr.Device): def __init__( self, sim = False, promProg = False, + rocev2 = False, + dcqcn = True, **kwargs): super().__init__(**kwargs) @@ -52,9 +55,18 @@ def __init__( self, self.add(udp.UdpEngine( offset = 0x0011_0000, numSrv = 2, + numClt = 1 if rocev2 else 0, enabled = not sim, )) + if rocev2: + self.add(roce.RoCEv2AxiStreamRdma( + offset = 0x0015_0000, + dcqcn = dcqcn, + expand = False, + enabled = not sim, + )) + for i in range(2): self.add(rssi.RssiCore( name = f'FwRudpServer[{i}]', @@ -79,4 +91,4 @@ def __init__( self, self.add(xceiver.Sfp( offset = 0x0020_2000, enabled = not sim, - )) \ No newline at end of file + )) diff --git a/firmware/releases.yaml b/firmware/releases.yaml index 0aefd12..8ebc2c2 100644 --- a/firmware/releases.yaml +++ b/firmware/releases.yaml @@ -1,23 +1,10 @@ GitBase: .. -TopRoguePackage: simple_10gbe_rudp_kcu105_example - -RoguePackages: - - submodules/surf/python - - python - -RogueConfig: - - ../software/config - -RogueScripts: - - ../software/scripts/devGui.py - Targets: Simple10GbeRudpKcu105Example: ImageDir: targets/Simple10GbeRudpKcu105Example/images Extensions: - - bit - mcs - ltx @@ -27,4 +14,4 @@ Releases: Targets: - Simple10GbeRudpKcu105Example Types: - - Rogue + - FW_only diff --git a/firmware/shared/rtl/App.vhd b/firmware/shared/rtl/App.vhd old mode 100644 new mode 100755 index 3337b60..dd1bd03 --- a/firmware/shared/rtl/App.vhd +++ b/firmware/shared/rtl/App.vhd @@ -18,12 +18,19 @@ use ieee.std_logic_1164.all; library surf; use surf.StdRtlPkg.all; use surf.AxiStreamPkg.all; +use surf.SsiPkg.all; use surf.AxiLitePkg.all; +use surf.RssiPkg.all; + +library work; +use work.CorePkg.all; entity App is generic ( - TPD_G : time := 1 ns; - SIMULATION_G : boolean := false); + TPD_G : time := 1 ns; + AXIS_CLK_FREQ_G : real := 156.25E+6; + ROCEV2_EN_G : boolean := false; + SIMULATION_G : boolean := false); port ( -- Clock and Reset axilClk : in sl; @@ -33,6 +40,9 @@ entity App is ibRudpSlave : in AxiStreamSlaveType; obRudpMaster : in AxiStreamMasterType; obRudpSlave : out AxiStreamSlaveType; + -- RDMA AXI-Stream Interface + rdmaMaster : out AxiStreamMasterType; + rdmaSlave : in AxiStreamSlaveType := AXI_STREAM_SLAVE_FORCE_C; -- AXI-Lite Interface axilReadMaster : in AxiLiteReadMasterType; axilReadSlave : out AxiLiteReadSlaveType; @@ -42,10 +52,12 @@ end App; architecture mapping of App is - constant TX_INDEX_C : natural := 0; - constant MEM_INDEX_C : natural := 1; + constant TX_INDEX_C : natural := 0; + constant MEM_INDEX_C : natural := 1; + constant PRBS_INDEX_C : natural := 2; + constant RDMA_MON_INDEX_C : natural := 3; - constant NUM_AXIL_MASTERS_C : positive := 2; + constant NUM_AXIL_MASTERS_C : positive := 4; constant XBAR_CONFIG_C : AxiLiteCrossbarMasterConfigArray(NUM_AXIL_MASTERS_C-1 downto 0) := genAxiLiteConfig(NUM_AXIL_MASTERS_C, x"8000_0000", 20, 16); @@ -54,6 +66,9 @@ architecture mapping of App is signal axilReadMasters : AxiLiteReadMasterArray(NUM_AXIL_MASTERS_C-1 downto 0); signal axilReadSlaves : AxiLiteReadSlaveArray(NUM_AXIL_MASTERS_C-1 downto 0) := (others => AXI_LITE_READ_SLAVE_EMPTY_SLVERR_C); + signal prbsAxisMaster : AxiStreamMasterType; + signal prbsAxisSlave : AxiStreamSlaveType; + begin ------------------------------- @@ -121,4 +136,81 @@ begin axiWriteMaster => axilWriteMasters(MEM_INDEX_C), axiWriteSlave => axilWriteSlaves(MEM_INDEX_C)); + GEN_ROCEV2_APP_LOGIC : if ROCEV2_EN_G generate + + -------------------------------- + -- PRBS payload source (host/AXI-Lite-controlled) + -------------------------------- + U_SsiPrbsTx : entity surf.SsiPrbsTx + generic map ( + TPD_G => TPD_G, + AXI_EN_G => '1', + AXI_DEFAULT_PKT_LEN_G => toSlv(509, 32), -- 510 words x 8B = 4080B raw; +16B packetizer = 4096B SEND + GEN_SYNC_FIFO_G => true, + PRBS_SEED_SIZE_G => 8*RDMA_AXIS_CONFIG_C.TDATA_BYTES_C, + PRBS_INCREMENT_G => false, + MASTER_AXI_STREAM_CONFIG_G => RDMA_AXIS_CONFIG_C) + port map ( + -- Master Port (mAxisClk domain) + mAxisClk => axilClk, + mAxisRst => axilRst, + mAxisMaster => prbsAxisMaster, + mAxisSlave => prbsAxisSlave, + -- Trigger Signal (locClk domain); AXI_EN_G='1' -> host owns trig/length + locClk => axilClk, + locRst => axilRst, + -- AXI-Lite Interface + axilReadMaster => axilReadMasters(PRBS_INDEX_C), + axilReadSlave => axilReadSlaves(PRBS_INDEX_C), + axilWriteMaster => axilWriteMasters(PRBS_INDEX_C), + axilWriteSlave => axilWriteSlaves(PRBS_INDEX_C)); + + + -------------------------------- + -- AXI-Stream Monitor on the RDMA stream + -------------------------------- + U_RdmaAxisMon : entity surf.AxiStreamMonAxiL + generic map ( + TPD_G => TPD_G, + COMMON_CLK_G => true, + AXIS_CLK_FREQ_G => AXIS_CLK_FREQ_G, + AXIS_NUM_SLOTS_G => 1, + AXIS_CONFIG_G => RDMA_AXIS_CONFIG_C) + port map ( + -- AXIS Stream Interface + axisClk => axilClk, + axisRst => axilRst, + axisMasters(0) => prbsAxisMaster, + axisSlaves(0) => prbsAxisSlave, + -- AXI lite slave port for register access + axilClk => axilClk, + axilRst => axilRst, + sAxilWriteMaster => axilWriteMasters(RDMA_MON_INDEX_C), + sAxilWriteSlave => axilWriteSlaves(RDMA_MON_INDEX_C), + sAxilReadMaster => axilReadMasters(RDMA_MON_INDEX_C), + sAxilReadSlave => axilReadSlaves(RDMA_MON_INDEX_C)); + + U_Packetizer : entity surf.AxiStreamPacketizer2 + generic map ( + TPD_G => TPD_G, + MEMORY_TYPE_G => "block", + REG_EN_G => true, + CRC_MODE_G => "NONE", -- NONE because RoCEv2 always has a CRC + MAX_PACKET_BYTES_G => 4096, -- PMTU=4096 + INPUT_PIPE_STAGES_G => 1, + OUTPUT_PIPE_STAGES_G => 1) + port map ( + axisClk => axilClk, + axisRst => axilRst, + sAxisMaster => prbsAxisMaster, + sAxisSlave => prbsAxisSlave, + mAxisMaster => rdmaMaster, + mAxisSlave => rdmaSlave); + + end generate GEN_ROCEV2_APP_LOGIC; + + GEN_ROCEV2_TIEOFF : if (not ROCEV2_EN_G) generate + rdmaMaster <= AXI_STREAM_MASTER_INIT_C; + end generate GEN_ROCEV2_TIEOFF; + end mapping; diff --git a/firmware/shared/rtl/Core.vhd b/firmware/shared/rtl/Core.vhd index 06fcafc..e203616 100755 --- a/firmware/shared/rtl/Core.vhd +++ b/firmware/shared/rtl/Core.vhd @@ -31,66 +31,71 @@ use unisim.vcomponents.all; entity Core is generic ( - TPD_G : time := 1 ns; + TPD_G : time := 1 ns; BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := false; + DCQCN_EN_G : boolean := false; SIMULATION_G : boolean; ETH_BUILD_G : BuildEthType; IP_ADDR_G : slv(31 downto 0); DHCP_G : boolean); port ( -- Clock and Reset - axilClk : out sl; - axilRst : out sl; + axilClk : out sl; + axilRst : out sl; -- AXI-Stream Interface - ibRudpMaster : in AxiStreamMasterType; - ibRudpSlave : out AxiStreamSlaveType; - obRudpMaster : out AxiStreamMasterType; - obRudpSlave : in AxiStreamSlaveType; + ibRudpMaster : in AxiStreamMasterType; + ibRudpSlave : out AxiStreamSlaveType; + obRudpMaster : out AxiStreamMasterType; + obRudpSlave : in AxiStreamSlaveType; -- AXI-Lite Interface - axilReadMaster : out AxiLiteReadMasterType; - axilReadSlave : in AxiLiteReadSlaveType; - axilWriteMaster : out AxiLiteWriteMasterType; - axilWriteSlave : in AxiLiteWriteSlaveType; + axilReadMaster : out AxiLiteReadMasterType; + axilReadSlave : in AxiLiteReadSlaveType; + axilWriteMaster : out AxiLiteWriteMasterType; + axilWriteSlave : in AxiLiteWriteSlaveType; + -- RDMA AXI-Stream Interface (from App master, to Rudp slave) + rdmaMaster : in AxiStreamMasterType := AXI_STREAM_MASTER_INIT_C; + rdmaSlave : out AxiStreamSlaveType; -- I2C Ports - sfpTxDisL : out sl; - i2cRstL : out sl; - i2cScl : inout sl; - i2cSda : inout sl; + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; -- SYSMON Ports - vPIn : in sl; - vNIn : in sl; + vPIn : in sl; + vNIn : in sl; -- System Ports - extRst : in sl; - sysClk300P : in sl; - sysClk300N : in sl; - emcClk : in sl; - heartbeat : out sl; - phyReady : out sl; - rssiLinkUp : out slv(1 downto 0); + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + emcClk : in sl; + heartbeat : out sl; + phyReady : out sl; + rssiLinkUp : out slv(1 downto 0); -- Boot Memory Ports - flashCsL : out sl; - flashMosi : out sl; - flashMiso : in sl; - flashHoldL : out sl; - flashWp : out sl; + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; -- SFP ETH Ports - ethClkP : in sl; - ethClkN : in sl; - ethRxP : in sl; - ethRxN : in sl; - ethTxP : out sl; - ethTxN : out sl; + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; -- RJ45 ETH Ports - phyClkP : in sl; - phyClkN : in sl; - phyRxP : in sl; - phyRxN : in sl; - phyTxP : out sl; - phyTxN : out sl; - phyMdc : out sl; - phyMdio : inout sl; - phyRstN : out sl; - phyIrqN : in sl); + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); end Core; architecture mapping of Core is @@ -199,51 +204,56 @@ begin ETH_BUILD_G => ETH_BUILD_G, IP_ADDR_G => IP_ADDR_G, DHCP_G => DHCP_G, + ROCEV2_EN_G => ROCEV2_EN_G, + DCQCN_EN_G => DCQCN_EN_G, AXIL_BASE_ADDR_G => XBAR_CONFIG_C(ETH_INDEX_C).baseAddr) port map ( -- System Ports - extRst => extRst, - sysClk300P => sysClk300P, - sysClk300N => sysClk300N, + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, -- Ethernet Status - phyReady => phyReady, - rssiLinkUp => rssiLinkUp, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, -- Clock and Reset - axilClk => clk, - axilRst => rst, + axilClk => clk, + axilRst => rst, -- AXI-Stream Interface - ibRudpMaster => ibRudpMaster, - ibRudpSlave => ibRudpSlave, - obRudpMaster => obRudpMaster, - obRudpSlave => obRudpSlave, + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, -- Master AXI-Lite Interface - mAxilReadMaster => mAxilReadMaster, - mAxilReadSlave => mAxilReadSlave, - mAxilWriteMaster => mAxilWriteMaster, - mAxilWriteSlave => mAxilWriteSlave, + mAxilReadMaster => mAxilReadMaster, + mAxilReadSlave => mAxilReadSlave, + mAxilWriteMaster => mAxilWriteMaster, + mAxilWriteSlave => mAxilWriteSlave, -- Slave AXI-Lite Interfaces - sAxilReadMaster => axilReadMasters(ETH_INDEX_C), - sAxilReadSlave => axilReadSlaves(ETH_INDEX_C), - sAxilWriteMaster => axilWriteMasters(ETH_INDEX_C), - sAxilWriteSlave => axilWriteSlaves(ETH_INDEX_C), + sAxilReadMaster => axilReadMasters(ETH_INDEX_C), + sAxilReadSlave => axilReadSlaves(ETH_INDEX_C), + sAxilWriteMaster => axilWriteMasters(ETH_INDEX_C), + sAxilWriteSlave => axilWriteSlaves(ETH_INDEX_C), + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, -- SFP ETH Ports - ethClkP => ethClkP, - ethClkN => ethClkN, - ethRxP => ethRxP, - ethRxN => ethRxN, - ethTxP => ethTxP, - ethTxN => ethTxN, + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, -- RJ45 ETH Ports - phyClkP => phyClkP, - phyClkN => phyClkN, - phyRxP => phyRxP, - phyRxN => phyRxN, - phyTxP => phyTxP, - phyTxN => phyTxN, - phyMdc => phyMdc, - phyMdio => phyMdio, - phyRstN => phyRstN, - phyIrqN => phyIrqN); + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); end generate; diff --git a/firmware/shared/rtl/CorePkg.vhd b/firmware/shared/rtl/CorePkg.vhd index a1330fb..82d11d3 100755 --- a/firmware/shared/rtl/CorePkg.vhd +++ b/firmware/shared/rtl/CorePkg.vhd @@ -17,9 +17,14 @@ use ieee.std_logic_1164.all; library surf; use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.SsiPkg.all; package CorePkg is type BuildEthType is (SFP_10G_C, SFP_1G_C, RJ45_1G_C); + -- 64-bit RDMA payload AXI-Stream config (moved from App.vhd) + constant RDMA_AXIS_CONFIG_C : AxiStreamConfigType := ssiAxiStreamConfig(dataBytes => (64/8)); + end package CorePkg; diff --git a/firmware/shared/rtl/Rudp.vhd b/firmware/shared/rtl/Rudp.vhd index e5e0783..847dc75 100755 --- a/firmware/shared/rtl/Rudp.vhd +++ b/firmware/shared/rtl/Rudp.vhd @@ -30,55 +30,60 @@ use unisim.vcomponents.all; entity Rudp is generic ( - TPD_G : time := 1 ns; + TPD_G : time := 1 ns; ETH_BUILD_G : BuildEthType; IP_ADDR_G : slv(31 downto 0); DHCP_G : boolean; + ROCEV2_EN_G : boolean := false; + DCQCN_EN_G : boolean := false; AXIL_BASE_ADDR_G : slv(31 downto 0)); port ( -- System Ports - extRst : in sl; - sysClk300P : in sl; - sysClk300N : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; -- Ethernet Status - phyReady : out sl; - rssiLinkUp : out slv(1 downto 0); + phyReady : out sl; + rssiLinkUp : out slv(1 downto 0); -- Clock and Reset - axilClk : out sl; - axilRst : out sl; + axilClk : out sl; + axilRst : out sl; -- AXI-Stream Interface - ibRudpMaster : in AxiStreamMasterType; - ibRudpSlave : out AxiStreamSlaveType; - obRudpMaster : out AxiStreamMasterType; - obRudpSlave : in AxiStreamSlaveType; + ibRudpMaster : in AxiStreamMasterType; + ibRudpSlave : out AxiStreamSlaveType; + obRudpMaster : out AxiStreamMasterType; + obRudpSlave : in AxiStreamSlaveType; -- Master AXI-Lite Interface - mAxilReadMaster : out AxiLiteReadMasterType; - mAxilReadSlave : in AxiLiteReadSlaveType; - mAxilWriteMaster : out AxiLiteWriteMasterType; - mAxilWriteSlave : in AxiLiteWriteSlaveType; + mAxilReadMaster : out AxiLiteReadMasterType; + mAxilReadSlave : in AxiLiteReadSlaveType; + mAxilWriteMaster : out AxiLiteWriteMasterType; + mAxilWriteSlave : in AxiLiteWriteSlaveType; -- Slave AXI-Lite Interfaces - sAxilReadMaster : in AxiLiteReadMasterType; - sAxilReadSlave : out AxiLiteReadSlaveType; - sAxilWriteMaster : in AxiLiteWriteMasterType; - sAxilWriteSlave : out AxiLiteWriteSlaveType; + sAxilReadMaster : in AxiLiteReadMasterType; + sAxilReadSlave : out AxiLiteReadSlaveType; + sAxilWriteMaster : in AxiLiteWriteMasterType; + sAxilWriteSlave : out AxiLiteWriteSlaveType; + -- RDMA AXI-Stream Interface (slave; feeds the RoCEv2AxiStreamRdma wrapper) + rdmaMaster : in AxiStreamMasterType := AXI_STREAM_MASTER_INIT_C; + rdmaSlave : out AxiStreamSlaveType; -- SFP ETH Ports - ethClkP : in sl; - ethClkN : in sl; - ethRxP : in sl; - ethRxN : in sl; - ethTxP : out sl; - ethTxN : out sl; + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; -- RJ45 ETH Ports - phyClkP : in sl; - phyClkN : in sl; - phyRxP : in sl; - phyRxN : in sl; - phyTxP : out sl; - phyTxN : out sl; - phyMdc : out sl; - phyMdio : inout sl; - phyRstN : out sl; - phyIrqN : in sl); + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); end Rudp; architecture mapping of Rudp is @@ -87,8 +92,9 @@ architecture mapping of Rudp is constant UDP_INDEX_C : natural := 1; constant RSSI_INDEX_C : natural := 2; -- 2:3 constant AXIS_MON_INDEX_C : natural := 4; + constant ROCE_INDEX_C : natural := 5; - constant NUM_AXIL_MASTERS_C : positive := 5; + constant NUM_AXIL_MASTERS_C : positive := 6; constant XBAR_CONFIG_C : AxiLiteCrossbarMasterConfigArray(NUM_AXIL_MASTERS_C-1 downto 0) := genAxiLiteConfig(NUM_AXIL_MASTERS_C, AXIL_BASE_ADDR_G, 20, 16); @@ -99,7 +105,7 @@ architecture mapping of Rudp is constant CLK_FREQUENCY_C : real := ite((ETH_BUILD_G = SFP_10G_C), 156.25E+6, 125.0E+6); - -- UDP constants + -- UDP Server constants constant UDP_SRV_SRP_IDX_C : natural := 0; constant UDP_SRV_DATA_IDX_C : natural := 1; constant UDP_SRV_XVC_IDX_C : natural := 2; @@ -109,6 +115,12 @@ architecture mapping of Rudp is UDP_SRV_DATA_IDX_C => 8193, -- Streaming data UDP_SRV_XVC_IDX_C => 2542); -- Xilinx XVC + -- UDP Client constants + constant UDP_CLT_ROCE_IDX_C : natural := 0; + constant CLIENT_SIZE_C : positive := 1; + constant CLIENT_PORTS_C : PositiveArray(CLIENT_SIZE_C-1 downto 0) := ( + UDP_CLT_ROCE_IDX_C => 4791); -- RoCEv2 + -- RSSI constants constant RSSI_SIZE_C : positive := 1; -- Implementing only 1 VC per RSSI link constant AXIS_CONFIG_C : AxiStreamConfigArray(RSSI_SIZE_C-1 downto 0) := ( @@ -119,10 +131,15 @@ architecture mapping of Rudp is signal obMacMaster : AxiStreamMasterType; signal obMacSlave : AxiStreamSlaveType; - signal obServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0); - signal obServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0); - signal ibServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0); - signal ibServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0); + signal obServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal obServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); + signal ibServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal ibServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); + + signal obClientMasters : AxiStreamMasterArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal obClientSlaves : AxiStreamSlaveArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); + signal ibClientMasters : AxiStreamMasterArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal ibClientSlaves : AxiStreamSlaveArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); -- One RSSI per UDP port (which is why SERVER_SIZE_C used instead of SERVER_SIZE_C) signal rssiIbMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0); @@ -223,6 +240,7 @@ begin TPD_G => TPD_G, NUM_LANE_G => 1, PAUSE_EN_G => true, -- Enable ETH pause + ROCEV2_EN_G => ROCEV2_EN_G, -- Enable RoCEv2 EN_AXI_REG_G => true) -- Enable diagnostic AXI-Lite interface port map ( -- Local Configurations @@ -273,6 +291,7 @@ begin TPD_G => TPD_G, NUM_LANE_G => 1, PAUSE_EN_G => true, -- Enable ETH pause + ROCEV2_EN_G => ROCEV2_EN_G, EN_AXI_REG_G => true, -- Enable diagnostic AXI-Lite interface -- QUAD PLL Configurations USE_GTREFCLK_G => false, @@ -356,6 +375,7 @@ begin generic map ( TPD_G => TPD_G, STABLE_CLK_FREQ_G => 300.0E+6, + ROCEV2_EN_G => ROCEV2_EN_G, PAUSE_EN_G => false, EN_AXIL_REG_G => true, AXIS_CONFIG_G => EMAC_AXIS_CONFIG_C) @@ -402,17 +422,22 @@ begin U_UDP : entity surf.UdpEngineWrapper generic map ( -- Simulation Generics - TPD_G => TPD_G, + TPD_G => TPD_G, -- UDP Server Generics - SERVER_EN_G => true, -- UDP Server only - SERVER_SIZE_G => SERVER_SIZE_C, - SERVER_PORTS_G => SERVER_PORTS_C, + SERVER_EN_G => true, -- UDP Server only + SERVER_SIZE_G => SERVER_SIZE_C, + SERVER_PORTS_G => SERVER_PORTS_C, -- UDP Client Generics - CLIENT_EN_G => false, -- UDP Server only + CLIENT_EN_G => ROCEV2_EN_G, -- UDP Server only if not RoCE + CLIENT_SIZE_G => CLIENT_SIZE_C, + CLIENT_PORTS_G => CLIENT_PORTS_C, + CLIENT_EXT_CONFIG_G => false, -- General IPv4/ARP/DHCP Generics - DHCP_G => DHCP_G, - CLK_FREQ_G => CLK_FREQUENCY_C, - COMM_TIMEOUT_G => 10) -- Timeout used for ARP and DHCP + DHCP_G => DHCP_G, + CLK_FREQ_G => CLK_FREQUENCY_C, + DSCP_G => 0, -- p2p default: no DSCP marking (set Core.UdpEngine.Dscp=26/AF31 at runtime for a managed fabric) + ECN_G => "00", -- p2p default: Not-ECT, keeps host NIC DCQCN disengaged (set EcnFlag=ECT(0) at runtime for a managed fabric) + COMM_TIMEOUT_G => 10) -- Timeout used for ARP and DHCP port map ( -- Local Configurations localMac => localMac, @@ -427,6 +452,10 @@ begin obServerSlaves => obServerSlaves, ibServerMasters => ibServerMasters, ibServerSlaves => ibServerSlaves, + obClientMasters => obClientMasters, + obClientSlaves => obClientSlaves, + ibClientMasters => ibClientMasters, + ibClientSlaves => ibClientSlaves, -- AXI-Lite Interface axilReadMaster => axilReadMasters(UDP_INDEX_C), axilReadSlave => axilReadSlaves(UDP_INDEX_C), @@ -536,6 +565,38 @@ begin obRudpMaster <= rssiObMasters(1); rssiObSlaves(1) <= obRudpSlave; + --------------------------------------------------------------- + -- RoCEv2 Engine + --------------------------------------------------------------- + GEN_ROCE_ENGINE : if ROCEV2_EN_G generate + U_RoceEngineWrapper : entity surf.RoCEv2AxiStreamRdma + generic map ( + TPD_G => TPD_G, + DCQCN_EN_G => DCQCN_EN_G, + AXIS_CONFIG_G => RDMA_AXIS_CONFIG_C, + AXIL_BASE_ADDR_G => XBAR_CONFIG_C(ROCE_INDEX_C).baseAddr) + port map ( + roceClk => ethClk, + roceRst => ethRst, + -- Inbound PRBS payload + sAxisMaster => rdmaMaster, + sAxisSlave => rdmaSlave, + -- Interface to UDP Engine (port 4791) + obUdpMaster => obClientMasters(UDP_CLT_ROCE_IDX_C), + obUdpSlave => obClientSlaves(UDP_CLT_ROCE_IDX_C), + ibUdpMaster => ibClientMasters(UDP_CLT_ROCE_IDX_C), + ibUdpSlave => ibClientSlaves(UDP_CLT_ROCE_IDX_C), + -- Axi-Lite interface + axilReadMaster => axilReadMasters(ROCE_INDEX_C), + axilReadSlave => axilReadSlaves(ROCE_INDEX_C), + axilWriteMaster => axilWriteMasters(ROCE_INDEX_C), + axilWriteSlave => axilWriteSlaves(ROCE_INDEX_C)); + end generate GEN_ROCE_ENGINE; + + GEN_ROCE_TIEOFF : if (not ROCEV2_EN_G) generate + rdmaSlave <= AXI_STREAM_SLAVE_FORCE_C; -- force slave ready so every target elaborates + end generate GEN_ROCE_TIEOFF; + ------------------------ -- AXI Stream Monitoring ------------------------ diff --git a/firmware/shared/ruckus.tcl b/firmware/shared/ruckus.tcl index 2a32225..cb33773 100644 --- a/firmware/shared/ruckus.tcl +++ b/firmware/shared/ruckus.tcl @@ -1,6 +1,9 @@ # Load RUCKUS library source $::env(RUCKUS_PROC_TCL) +# Check for version 2023.1 of Vivado (or later) +if { [VersionCheck 2023.1] < 0 } {exit -1} + # Load Source Code loadSource -dir "$::DIR_PATH/rtl" loadIpCore -path "$::DIR_PATH/ip/SystemManagementCore.xci" diff --git a/firmware/submodules/ruckus b/firmware/submodules/ruckus index 4c60fea..684ecc6 160000 --- a/firmware/submodules/ruckus +++ b/firmware/submodules/ruckus @@ -1 +1 @@ -Subproject commit 4c60fea930cac15341d08a5fd14af4919c785b48 +Subproject commit 684ecc69ad92d8283d84be41021f8e8d9c238681 diff --git a/firmware/submodules/surf b/firmware/submodules/surf index 634c8d9..142903a 160000 --- a/firmware/submodules/surf +++ b/firmware/submodules/surf @@ -1 +1 @@ -Subproject commit 634c8d9656e7f6c99e52c7d943fb6f72e74a95fa +Subproject commit 142903ac6609ab002ca1da5b36c780081368ea04 diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile new file mode 100755 index 0000000..a4be7fd --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile @@ -0,0 +1,6 @@ +# Define the TOP_DIR path +export TOP_DIR = $(abspath $(PWD)/../..) + +# Use top level makefile +include ../shared_version.mk +include $(TOP_DIR)/submodules/ruckus/system_vivado.mk diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd new file mode 100755 index 0000000..5f1e3ed --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd @@ -0,0 +1,211 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Simple 10G-BASER Example +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; + +library work; +use work.CorePkg.all; + +entity RoCEv2_10GbeRudpKcu105Example is + generic ( + TPD_G : time := 1 ns; + BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := true; + DCQCN_EN_G : boolean := true; + SIMULATION_G : boolean := false; + IP_ADDR_G : slv(31 downto 0) := x"0A02A8C0"; -- 192.168.2.10 + DHCP_G : boolean := false); + port ( + -- I2C Ports + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; + -- XADC Ports + vPIn : in sl; + vNIn : in sl; + -- System Ports + emcClk : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + led : out slv(7 downto 0); + -- Boot Memory Ports + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; + -- SFP ETH Ports + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; + -- RJ45 ETH Ports + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); +end RoCEv2_10GbeRudpKcu105Example; + +architecture top_level of RoCEv2_10GbeRudpKcu105Example is + + signal heartbeat : sl; + signal phyReady : sl; + signal rssiLinkUp : slv(1 downto 0); + + -- Clock and Reset + signal axilClk : sl; + signal axilRst : sl; + + -- AXI-Stream: Stream Interface + signal ibRudpMaster : AxiStreamMasterType; + signal ibRudpSlave : AxiStreamSlaveType; + signal obRudpMaster : AxiStreamMasterType; + signal obRudpSlave : AxiStreamSlaveType; + + -- AXI-Lite: Register Access + signal axilReadMaster : AxiLiteReadMasterType; + signal axilReadSlave : AxiLiteReadSlaveType; + signal axilWriteMaster : AxiLiteWriteMasterType; + signal axilWriteSlave : AxiLiteWriteSlaveType; + + -- RDMA AXI-Stream (App master -> Core -> Rudp slave) + signal rdmaMaster : AxiStreamMasterType; + signal rdmaSlave : AxiStreamSlaveType; + +begin + + led(7) <= '1'; + led(6) <= '0'; + led(5) <= heartbeat; + led(4) <= axilRst; + led(3) <= not(axilRst); + led(2) <= rssiLinkUp(1); + led(1) <= rssiLinkUp(0); + led(0) <= phyReady; + + ----------------------- + -- Core Firmware Module + ----------------------- + U_Core : entity work.Core + generic map ( + TPD_G => TPD_G, + BUILD_INFO_G => BUILD_INFO_G, + ROCEV2_EN_G => ROCEV2_EN_G, + DCQCN_EN_G => DCQCN_EN_G, + SIMULATION_G => SIMULATION_G, + ETH_BUILD_G => SFP_10G_C, + IP_ADDR_G => IP_ADDR_G, + DHCP_G => DHCP_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave, + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, + -- I2C Ports + sfpTxDisL => sfpTxDisL, + i2cRstL => i2cRstL, + i2cScl => i2cScl, + i2cSda => i2cSda, + -- SYSMON Ports + vPIn => vPIn, + vNIn => vNIn, + -- System Ports + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, + emcClk => emcClk, + heartbeat => heartbeat, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, + -- Boot Memory Ports + flashCsL => flashCsL, + flashMosi => flashMosi, + flashMiso => flashMiso, + flashHoldL => flashHoldL, + flashWp => flashWp, + -- SFP ETH Ports + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, + -- RJ45 ETH Ports + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); + + ------------------------------ + -- Application Firmware Module + ------------------------------ + U_App : entity work.App + generic map ( + TPD_G => TPD_G, + AXIS_CLK_FREQ_G => 156.25E+6, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave); + +end top_level; diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl new file mode 100755 index 0000000..540b450 --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl @@ -0,0 +1,10 @@ +# Load RUCKUS environment and library +source $::env(RUCKUS_PROC_TCL) + +# Load shared and sub-module ruckus.tcl files +loadRuckusTcl $::env(TOP_DIR)/submodules/surf +loadRuckusTcl $::env(TOP_DIR)/shared + +# Load local source Code and constraints +loadSource -dir "$::DIR_PATH/hdl" +loadConstraints -dir "$::DIR_PATH/../Simple10GbeRudpKcu105Example/hdl" diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl new file mode 100755 index 0000000..938b225 --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl @@ -0,0 +1,13 @@ +############################################################################## +## This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## + +set format "mcs" +set inteface "SPIx8" +set size "512" diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile new file mode 100755 index 0000000..a4be7fd --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile @@ -0,0 +1,6 @@ +# Define the TOP_DIR path +export TOP_DIR = $(abspath $(PWD)/../..) + +# Use top level makefile +include ../shared_version.mk +include $(TOP_DIR)/submodules/ruckus/system_vivado.mk diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd new file mode 100755 index 0000000..df2f1a3 --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd @@ -0,0 +1,210 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Simple 1000baseX/Full Example +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; + +library work; +use work.CorePkg.all; + +entity RoCEv2_1GbeRudpKcu105Example is + generic ( + TPD_G : time := 1 ns; + BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := false; + SIMULATION_G : boolean := false; + IP_ADDR_G : slv(31 downto 0) := x"0A02A8C0"; -- 192.168.2.10 + DHCP_G : boolean := false); + port ( + -- I2C Ports + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; + -- XADC Ports + vPIn : in sl; + vNIn : in sl; + -- System Ports + emcClk : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + led : out slv(7 downto 0); + -- Boot Memory Ports + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; + -- SFP ETH Ports + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; + -- RJ45 ETH Ports + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); +end RoCEv2_1GbeRudpKcu105Example; + +architecture top_level of RoCEv2_1GbeRudpKcu105Example is + + signal heartbeat : sl; + signal phyReady : sl; + signal rssiLinkUp : slv(1 downto 0); + + -- Clock and Reset + signal axilClk : sl; + signal axilRst : sl; + + -- AXI-Stream: Stream Interface + signal ibRudpMaster : AxiStreamMasterType; + signal ibRudpSlave : AxiStreamSlaveType; + signal obRudpMaster : AxiStreamMasterType; + signal obRudpSlave : AxiStreamSlaveType; + + -- AXI-Lite: Register Access + signal axilReadMaster : AxiLiteReadMasterType; + signal axilReadSlave : AxiLiteReadSlaveType; + signal axilWriteMaster : AxiLiteWriteMasterType; + signal axilWriteSlave : AxiLiteWriteSlaveType; + + -- RDMA AXI-Stream (App master -> Core -> Rudp slave) + signal rdmaMaster : AxiStreamMasterType; + signal rdmaSlave : AxiStreamSlaveType; + + +begin + + led(7) <= '1'; + led(6) <= '0'; + led(5) <= heartbeat; + led(4) <= axilRst; + led(3) <= not(axilRst); + led(2) <= rssiLinkUp(1); + led(1) <= rssiLinkUp(0); + led(0) <= phyReady; + + ----------------------- + -- Core Firmware Module + ----------------------- + U_Core : entity work.Core + generic map ( + TPD_G => TPD_G, + BUILD_INFO_G => BUILD_INFO_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G, + ETH_BUILD_G => SFP_1G_C, + IP_ADDR_G => IP_ADDR_G, + DHCP_G => DHCP_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave, + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, + -- I2C Ports + sfpTxDisL => sfpTxDisL, + i2cRstL => i2cRstL, + i2cScl => i2cScl, + i2cSda => i2cSda, + -- SYSMON Ports + vPIn => vPIn, + vNIn => vNIn, + -- System Ports + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, + emcClk => emcClk, + heartbeat => heartbeat, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, + -- Boot Memory Ports + flashCsL => flashCsL, + flashMosi => flashMosi, + flashMiso => flashMiso, + flashHoldL => flashHoldL, + flashWp => flashWp, + -- SFP ETH Ports + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, + -- RJ45 ETH Ports + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); + + ------------------------------ + -- Application Firmware Module + ------------------------------ + U_App : entity work.App + generic map ( + TPD_G => TPD_G, + AXIS_CLK_FREQ_G => 125.0E+6, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave); + +end top_level; diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl new file mode 100755 index 0000000..6890cbf --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl @@ -0,0 +1,10 @@ +# Load RUCKUS environment and library +source $::env(RUCKUS_PROC_TCL) + +# Load shared and sub-module ruckus.tcl files +loadRuckusTcl $::env(TOP_DIR)/submodules/surf +loadRuckusTcl $::env(TOP_DIR)/shared + +# Load local source Code and constraints +loadSource -dir "$::DIR_PATH/hdl" +loadConstraints -dir "$::DIR_PATH/../Simple1GbeRudpKcu105Example/hdl" diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl new file mode 100755 index 0000000..938b225 --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl @@ -0,0 +1,13 @@ +############################################################################## +## This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## + +set format "mcs" +set inteface "SPIx8" +set size "512" diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile new file mode 100755 index 0000000..a4be7fd --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile @@ -0,0 +1,6 @@ +# Define the TOP_DIR path +export TOP_DIR = $(abspath $(PWD)/../..) + +# Use top level makefile +include ../shared_version.mk +include $(TOP_DIR)/submodules/ruckus/system_vivado.mk diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd new file mode 100755 index 0000000..aabc6d1 --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd @@ -0,0 +1,209 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Simple 1000baseT/Full Example +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; + +library work; +use work.CorePkg.all; + +entity RoCEv2_Rj45RudpKcu105Example is + generic ( + TPD_G : time := 1 ns; + BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := false; + SIMULATION_G : boolean := false; + IP_ADDR_G : slv(31 downto 0) := x"0A02A8C0"; -- 192.168.2.10 + DHCP_G : boolean := false); + port ( + -- I2C Ports + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; + -- XADC Ports + vPIn : in sl; + vNIn : in sl; + -- System Ports + emcClk : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + led : out slv(7 downto 0); + -- Boot Memory Ports + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; + -- SFP ETH Ports + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; + -- RJ45 ETH Ports + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); +end RoCEv2_Rj45RudpKcu105Example; + +architecture top_level of RoCEv2_Rj45RudpKcu105Example is + + signal heartbeat : sl; + signal phyReady : sl; + signal rssiLinkUp : slv(1 downto 0); + + -- Clock and Reset + signal axilClk : sl; + signal axilRst : sl; + + -- AXI-Stream: Stream Interface + signal ibRudpMaster : AxiStreamMasterType; + signal ibRudpSlave : AxiStreamSlaveType; + signal obRudpMaster : AxiStreamMasterType; + signal obRudpSlave : AxiStreamSlaveType; + + -- AXI-Lite: Register Access + signal axilReadMaster : AxiLiteReadMasterType; + signal axilReadSlave : AxiLiteReadSlaveType; + signal axilWriteMaster : AxiLiteWriteMasterType; + signal axilWriteSlave : AxiLiteWriteSlaveType; + + -- RDMA AXI-Stream (App master -> Core -> Rudp slave) + signal rdmaMaster : AxiStreamMasterType; + signal rdmaSlave : AxiStreamSlaveType; + +begin + + led(7) <= '1'; + led(6) <= '0'; + led(5) <= heartbeat; + led(4) <= axilRst; + led(3) <= not(axilRst); + led(2) <= rssiLinkUp(1); + led(1) <= rssiLinkUp(0); + led(0) <= phyReady; + + ----------------------- + -- Core Firmware Module + ----------------------- + U_Core : entity work.Core + generic map ( + TPD_G => TPD_G, + BUILD_INFO_G => BUILD_INFO_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G, + ETH_BUILD_G => RJ45_1G_C, + IP_ADDR_G => IP_ADDR_G, + DHCP_G => DHCP_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave, + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, + -- I2C Ports + sfpTxDisL => sfpTxDisL, + i2cRstL => i2cRstL, + i2cScl => i2cScl, + i2cSda => i2cSda, + -- SYSMON Ports + vPIn => vPIn, + vNIn => vNIn, + -- System Ports + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, + emcClk => emcClk, + heartbeat => heartbeat, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, + -- Boot Memory Ports + flashCsL => flashCsL, + flashMosi => flashMosi, + flashMiso => flashMiso, + flashHoldL => flashHoldL, + flashWp => flashWp, + -- SFP ETH Ports + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, + -- RJ45 ETH Ports + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); + + ------------------------------ + -- Application Firmware Module + ------------------------------ + U_App : entity work.App + generic map ( + TPD_G => TPD_G, + AXIS_CLK_FREQ_G => 125.0E+6, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- RDMA AXI-Stream Interface + rdmaMaster => rdmaMaster, + rdmaSlave => rdmaSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave); + +end top_level; diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl new file mode 100755 index 0000000..fb03bbd --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl @@ -0,0 +1,15 @@ +# Load RUCKUS environment and library +source $::env(RUCKUS_PROC_TCL) + +# Load shared and sub-module ruckus.tcl files +loadRuckusTcl $::env(TOP_DIR)/submodules/surf +loadRuckusTcl $::env(TOP_DIR)/shared + +# Load local source Code and constraints +loadSource -dir "$::DIR_PATH/hdl" +loadConstraints -dir "$::DIR_PATH/../SimpleRj45RudpKcu105Example/hdl" + +# Modified the .XDC property +set_property PROCESSING_ORDER {EARLY} [get_files {GigEthLvdsUltraScaleCore.xdc}] +set_property SCOPED_TO_REF {GigEthLvdsUltraScaleCore} [get_files {GigEthLvdsUltraScaleCore.xdc}] +set_property SCOPED_TO_CELLS {U0} [get_files {GigEthLvdsUltraScaleCore.xdc}] diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl new file mode 100755 index 0000000..938b225 --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl @@ -0,0 +1,13 @@ +############################################################################## +## This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## + +set format "mcs" +set inteface "SPIx8" +set size "512" diff --git a/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl index cb943a4..bde61f5 100644 --- a/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl +++ b/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl @@ -1,9 +1,6 @@ # Load RUCKUS environment and library source $::env(RUCKUS_PROC_TCL) -# Check for version 2023.1 of Vivado (or later) -if { [VersionCheck 2023.1] < 0 } {exit -1} - # Load shared and sub-module ruckus.tcl files loadRuckusTcl $::env(TOP_DIR)/submodules/surf loadRuckusTcl $::env(TOP_DIR)/shared diff --git a/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl index 73dd252..39c6bb1 100755 --- a/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl +++ b/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl @@ -1,9 +1,6 @@ # Load RUCKUS environment and library source $::env(RUCKUS_PROC_TCL) -# Check for version 2023.1 of Vivado (or later) -if { [VersionCheck 2023.1] < 0 } {exit -1} - # Load shared and sub-module ruckus.tcl files loadRuckusTcl $::env(TOP_DIR)/submodules/surf loadRuckusTcl $::env(TOP_DIR)/shared diff --git a/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl b/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl index 37da0de..80fc595 100755 --- a/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl +++ b/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl @@ -1,9 +1,6 @@ # Load RUCKUS environment and library source $::env(RUCKUS_PROC_TCL) -# Check for version 2023.1 of Vivado (or later) -if { [VersionCheck 2023.1] < 0 } {exit -1} - # Load shared and sub-module ruckus.tcl files loadRuckusTcl $::env(TOP_DIR)/submodules/surf loadRuckusTcl $::env(TOP_DIR)/shared diff --git a/firmware/targets/shared_version.mk b/firmware/targets/shared_version.mk index 6e9398f..d76c28f 100755 --- a/firmware/targets/shared_version.mk +++ b/firmware/targets/shared_version.mk @@ -1,5 +1,5 @@ -# Define Firmware Version: v2.19.0.0 -export PRJ_VERSION = 0x02190000 +# Define Firmware Version: v3.0.0.0 +export PRJ_VERSION = 0x03000000 # Define target output target: prom diff --git a/software/scripts/rocev2PrbsTest.py b/software/scripts/rocev2PrbsTest.py new file mode 100644 index 0000000..23f8ec8 --- /dev/null +++ b/software/scripts/rocev2PrbsTest.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- +import setupLibPaths # MUST be first — adds firmware/python + surf/python to sys.path + +import sys +import time +import argparse + +import pyrogue +import pyrogue.pydm + +import rocev2_10gbe_rudp_kcu105_example as roceBoard + +################################################################# + +if __name__ == "__main__": + + # Convert SIGTERM (terminal/WM close) to KeyboardInterrupt so the + # `with Root(...)` block unwinds into Root.stop() and tears down the FPGA QP. + # SIGINT/exceptions already unwind; SIGKILL relies on the FW auto-reset backstop. + import signal as _signal + + def _sigterm(_signum, _frame): + raise KeyboardInterrupt + + _signal.signal(_signal.SIGTERM, _sigterm) + + # Convert str to bool + argBool = lambda s: s.lower() in ['true', 't', 'yes', '1'] + + # Set the argument parser + parser = argparse.ArgumentParser( + description='RoCEv2 PRBS pass/fail launcher (+ optional PyDM GUI)') + + # Add arguments + parser.add_argument( + "--ip", + type = str, + required = False, + default = '192.168.2.10', + help = "IP address", + ) + + parser.add_argument( + "--pollEn", + type = argBool, + required = False, + default = True, + help = "Enable auto-polling", + ) + + parser.add_argument( + "--initRead", + type = argBool, + required = False, + default = True, + help = "Enable read all variables at start", + ) + + parser.add_argument( + "--guiType", + type = str, + required = False, + default = 'PyDM', + help = "Sets the GUI type (PyDM or None)", + ) + + parser.add_argument( + "--zmqSrvPort", + type = int, + required = False, + default = 9099, + help = "Zeromq server port (set to zero if you want it dynamic)", + ) + + parser.add_argument( + "--target", + required = False, + default = 1000, + type = int, + help = "Number of PRBS frames to receive before declaring PASS " + "(continuous event-driven dispatch)", + ) + + parser.add_argument( + "--trigRate", + required = False, + default = None, + type = float, + help = "PRBS packet rate in Hz (SsiPrbsTx.TrigRate). Default (unset) " + "auto-selects 2.5e4 for an mlx5 HW NIC and 5e3 for softRoCE (rxe0): " + "the Python PrbsRx consumer (and the rogue zero-copy recv-slot " + "re-post) saturates near 25-30 kHz (~100 MB/s) on an mlx5 NIC and far " + "lower on a kernel software responder, well below 10GbE line rate, so " + "above it PRBS errors appear -- host-stack limited, NOT a FW issue (the " + "FW reports successful completions far beyond this rate). Set 0 to " + "free-run the FW at full line rate; the link streams at line rate but " + "the PrbsRx validator cannot keep up and will report errors. Full-rate " + "per-frame validation needs a faster (non-Python) host consumer.", + ) + + parser.add_argument( + "--minRnrTimer", + required = False, + default = 12, + type = int, + help = "IB min_rnr_timer code for native FW<->NIC flow control (12=0.64ms, " + "1=0.01ms, 31=491ms). RDMA-SEND-with-immediate is two-sided, so when the " + "host recv queue drains the NIC RNR-NAKs the FPGA, which backs off this long " + "and retries — self-pacing the source to the host consume rate with NO " + "software credit writes in the real-time path. Too small storms RNR NAKs; " + "too large collapses throughput. Sweep 8..16 to find the knee.", + ) + + parser.add_argument( + "--p2p", + type = argBool, + required = False, + default = True, + help = "Point-to-point bring-up switch (DEFAULT True for the switchless " + "bench): forces Not-ECT egress + no DSCP marking and bypasses FW DCQCN " + "(DcqcnBypass=True). On an mlx5 HW NIC it also forces minimal RNR " + "backoff (minRnrTimer=1), overriding any explicit --minRnrTimer; on " + "softRoCE the RNR backoff is left at --minRnrTimer (minimal backoff " + "storms a software responder). Pass --p2p false for a managed ECN " + "fabric, then tune --dscp/--ecn/--enableDcqcn.", + ) + + parser.add_argument( + "--dscp", + type = int, + required = False, + default = 26, + help = "Managed-fabric only (ignored when --p2p true): IP-header DSCP for " + "TX packets (0-63). Default 26 = AF31; set to match the switch " + "lossless/ECN traffic class.", + ) + + parser.add_argument( + "--ecn", + type = int, + required = False, + default = 2, + help = "Managed-fabric only (ignored when --p2p true): IP-header ECN field " + "(0=Not-ECT, 1=ECT(1), 2=ECT(0)/b\"10\", 3=CE). Default 2 = ECT(0) " + "opts the flow into the fabric's ECN/DCQCN congestion control.", + ) + + parser.add_argument( + "--enableDcqcn", + type = argBool, + required = False, + default = True, + help = "Managed-fabric only (ignored when --p2p true): run FW DCQCN " + "congestion control (False bypasses it via DcqcnBypass).", + ) + + parser.add_argument( + "--roceGidIndex", + type = int, + required = False, + default = None, + help = "RoCEv2 GID index. If omitted, auto-detected as the RoCE v2 " + "IPv4 GID on --ip's subnet (pass explicitly to override)", + ) + + parser.add_argument( + "--roceDevice", + type = str, + required = False, + default = 'mlx5_0', + help = "ibverbs device name (mlx5_0 = HW NIC, rxe0 = softRoCE)", + ) + + parser.add_argument( + "--timeout", + type = float, + required = False, + default = 10.0, + help = "Seconds to poll for completion before declaring FAIL", + ) + + # Get the arguments + args = parser.parse_args() + + ################################################################# + # Validate / map CLI args before constructing the Root + ################################################################# + + # softRoCE escape hatch — warn loudly but proceed + isSoftRoce = not args.roceDevice.startswith('mlx5') + if isSoftRoce: + print( + f"WARNING: --roceDevice='{args.roceDevice}' is not an mlx5_* HW NIC; " + f"proceeding anyway (softRoCE / bench escape hatch).", + file=sys.stderr, + ) + + # --trigRate default is device-dependent: an mlx5 HW NIC sustains ~2.5e4 Hz of clean + # per-frame validation, but a kernel software responder (rxe0) is far slower. Pick a + # safe starting rate when the user did not pass --trigRate (tune to the knee with a + # sweep). An explicit value — including 0 = free-run line rate — is always honored. + if args.trigRate is None: + args.trigRate = 2.5e4 if not isSoftRoce else 5.0e3 + print( + f"NOTICE: --trigRate unset; defaulting to {args.trigRate:g} Hz for " + f"{'mlx5 HW NIC' if not isSoftRoce else 'softRoCE (tune to the knee)'}.", + file=sys.stderr, + ) + + # --target is the number of frames to receive before PASS; must be positive. + if args.target < 1: + print(f"ERROR: --target={args.target} must be >= 1.", file=sys.stderr) + sys.exit(1) + + # RoCEv2 GID index: explicit --roceGidIndex overrides; -1 lets RoCEv2Server + # auto-detect the RoCE v2 IPv4 GID on --ip's subnet (the index drifts across + # FPGA reloads, so it is detected fresh each run). + gidIndex = args.roceGidIndex if args.roceGidIndex is not None else -1 + + # Build the host-NIC config from the CLI args. The cfg defaults + # (maxPayload=4096, pmtu=MTU_4096) fix the 4096B framing — host recv buffer + # = full PMTU = FW MaxSize cap — so only the device/GID knobs are set here. + rocev2Cfg = pyrogue.protocols.RoCEv2ServerCfg( + ip = args.ip, + deviceName = args.roceDevice, + gidIndex = gidIndex, + ) + + # Transport / QP-tuning config. The Root forwards the single instance into BOTH + # engine.setupConnection() and server.completeConnection() so the FPGA and host + # cannot drift. --minRnrTimer is the native RNR backoff (FW<->NIC flow control); + # pmtu/rnrRetry/retryCount keep their proven acceptance-gate defaults. + # + # --p2p forces minimal RNR backoff (code 1) on an mlx5 HW NIC, overriding any + # --minRnrTimer; log an audit line if a non-default --minRnrTimer was also passed. + # (The Not-ECT + DcqcnBypass halves are applied in Root.start() from the args + # forwarded below, independent of the RNR timer.) + # + # softRoCE is the exception: keep the (larger) --minRnrTimer even under --p2p. A + # kernel software responder cannot keep its recv queue armed, and a 0.01ms backoff + # turns RNR-NAKs into a CPU-saturating retry storm that collapses throughput and + # stalls completions. The p2p egress posture still applies; only the RNR timer differs. + minRnrTimer = args.minRnrTimer + if args.p2p and not isSoftRoce: + if args.minRnrTimer != 12: + print( + f"NOTICE: --p2p overrode --minRnrTimer={args.minRnrTimer} to 1 " + f"(minimal RNR backoff for point-to-point).", + file=sys.stderr, + ) + minRnrTimer = 1 + elif args.p2p and isSoftRoce: + print( + f"NOTICE: softRoCE ({args.roceDevice}) — keeping minRnrTimer=" + f"{minRnrTimer} (NOT forcing code 1); p2p egress posture still applies.", + file=sys.stderr, + ) + + transportCfg = pyrogue.protocols.RoCEv2TransportCfg( + minRnrTimer = minRnrTimer, + ) + + ################################################################# + + with roceBoard.Root( + rocev2Cfg = rocev2Cfg, + transportCfg = transportCfg, + p2p = args.p2p, + dscp = args.dscp, + ecn = args.ecn, + enableDcqcn = args.enableDcqcn, + pollEn = args.pollEn, + initRead = args.initRead, + zmqSrvPort = args.zmqSrvPort, + ) as root: + + # Root.start() already ran the host<->FPGA bring-up; the RoCEv2 engine is + # connected-but-idle here, and the egress ECN/DSCP + DCQCN-bypass posture + # (p2p vs managed fabric) was applied inside start() from the args above. + # Arming the source (DispatchEnable/TxEn) is below. + rx = root.rdmaRx + + # MR parameters from the RoCEv2Server local variables. The on-wire RDMA SEND = + # one full PMTU (= host maxPayload = FW MaxSize cap). The FW AxiStreamPacketizer2 + # adds a 16B hdr+tail per frame, so the raw PRBS payload is one PMTU minus that + # overhead and raw + overhead = the PMTU-sized SEND. + PKTZR_OVERHEAD = 16 # AxiStreamPacketizer2 header word + tail word (CRC_MODE_G="NONE") + rx_queue_depth = rx.RxQueueDepth.get() + max_payload = rx.MaxPayload.get() + mr_len = rx_queue_depth * max_payload + Len = max_payload # on-wire SEND size (packetized frame) + raw_payload = max_payload - PKTZR_OVERHEAD # raw PRBS payload seen by PrbsRx after depacketize + remQpn = rx.HostQpn.get() + locKey = rx.FpgaLkey.get() + + prbs = root.App.SsiPrbsTx + dma = root.Core.RoCEv2AxiStreamRdma.Core + dcqcn = root.Core.RoCEv2AxiStreamRdma.Dcqcn # FW telemetry: Rc/Rt/CnpCounter (RO, pollInterval=1) + + # PRBS word size in bytes (SsiPrbsTx.WordSize = PRBS_SEED_SIZE_G bits), read + # from the FW so the host tracks its config. PacketLength counts whole words, + # so the raw PRBS payload (PMTU minus packetizer overhead) must be a multiple of it. + word_bytes = prbs.WordSize.get() // 8 + + if raw_payload % word_bytes != 0: + print( + f"ERROR: raw payload={raw_payload} (maxPayload {max_payload} - {PKTZR_OVERHEAD}B " + f"packetizer overhead) is not a multiple of the {word_bytes}-byte PRBS word " + f"— incompatible FW PRBS_SEED_SIZE_G.", + file=sys.stderr, + ) + sys.exit(1) + + # The host recv-WR buffer (maxPayload = full PMTU) must hold the LARGEST SEND + # the FW can dispatch, else a too-large SEND overflows it (Local Length Error + # -> receiver NAK -> UnsuccessCounter). FW per-SEND cap = the RO MaxSize + # register (MAX_BEATS_C*32 = one PMTU). + fw_max_send = dma.MaxSize.get() + if max_payload < fw_max_send: + print( + f"ERROR: host maxPayload={max_payload} < FW MaxSize={fw_max_send}; recv-WR " + f"buffer cannot hold the largest SEND (FW MaxSize exceeds maxPayload).", + file=sys.stderr, + ) + sys.exit(1) + if Len > fw_max_send: + print( + f"ERROR: maxPayload={Len} exceeds the FW per-SEND cap MaxSize={fw_max_send} " + f"(MAX_BEATS_C*32).", + file=sys.stderr, + ) + sys.exit(1) + + # ---------------------------------------------------------------- + # Configure the PRBS source + DMA dispatch registers + # ---------------------------------------------------------------- + # PacketLength in PRBS words: (raw_payload // word_bytes) - 1, sized so the + # packetized SEND lands at exactly one PMTU. Changeable LIVE in the GUI — the FW + # frames each SEND from the inbound tLast (no Len register), partial beats OK. + prbs.PacketLength.set(raw_payload // word_bytes - 1) + + # PRBS packet rate. TrigDly=0 free-runs at full line rate; a positive + # --trigRate throttles the source so the host receive path keeps up + # (avoids overrun + PRBS continuity errors during a continuous run). + if args.trigRate > 0: + prbs.TrigRate.set(args.trigRate) + else: + prbs.TrigDly.set(0) + + dma.LKey.set(locKey) + dma.SQpn.set(remQpn) + dma.AddrWrapCount.set(mr_len // Len) + # dma.DQpn left at default 0 — UD-datagram field, unused by the RC SEND path. + # RKey/RemAddr are legacy RETH registers; RDMA-SEND drives rAddr/rKey to 0 in + # the FW, so they are intentionally not configured here. + + # ---------------------------------------------------------------- + # Dual counter reset (FW + host) — RemoteCommand toggles, never .set(0/1/0) + # ---------------------------------------------------------------- + dma.ResetCounters() # zeroes FW SuccessCounter/UnsuccessCounter + root.CountReset() # zeroes host PrbsRx rxErrors/rxCount/rxBytes + + # ---------------------------------------------------------------- + # Event-driven continuous run. DispatchEnable arms the FW dispatcher; TxEn + # free-runs the PRBS source (AXI_EN_G='1' => host owns the trigger). With both + # set the FW issues one RDMA SEND per complete buffered PRBS packet — no + # per-frame poke. + # + # Flow control is entirely native FW<->NIC (no software credit feed): a full + # host recv queue RNR-NAKs the FPGA; the blue-rdma SQ stalls/retries + # (rnr_retry=7, min_rnr_timer=--minRnrTimer), backpressuring the dispatcher -> + # repack FIFO -> PRBS source. The host only posts/consumes recv-WRs at its pace. + # ---------------------------------------------------------------- + print(f"Streaming until rxCount >= {args.target} ({raw_payload}B raw / {Len}B on-wire " + f"per frame, native RNR flow control, minRnrTimer={minRnrTimer}, " + f"p2p={args.p2p})...") + + # Line-rate run (--trigRate <= 0, same condition that set TrigDly=0): blind the + # host payload validator. The Python PrbsRx consumer saturates ~25-30 kHz and + # cannot keep up at line rate, so per-frame checking would report spurious + # errors; throughput is gated on FW MonBandwidth instead of rxErrors. checkPayload + # is a live rogue RW LocalVariable, so this propagates to the C++ engine at + # runtime. Throttled runs (--trigRate > 0) keep the default (True), unchanged. + if args.trigRate <= 0: + print("NOTICE: line-rate run — disabling host PrbsRx.checkPayload " + "(rxErrors NOT gated; FW MonBandwidth is the throughput gate).", + file=sys.stderr) + root.PrbsRx.checkPayload.set(False) + + dma.DispatchEnable.set(True) + prbs.TxEn.set(True) + + # ---------------------------------------------------------------- + # Poll rxCount to the target — the receiver fills continuously + # ---------------------------------------------------------------- + # FW-telemetry trajectory sampler: snapshot (t_rel, Rc, Rt, CnpCounter, + # MonBandwidth) into `traj` at a ~0.5 s cadence so a line-rate run captures the + # Rc collapse trajectory (e.g. Rc LINE/2->Rmin under DCQCN throttle), not just an + # endpoint. A monotonic `nextSample` deadline keeps the 0.05 s rxCount poll + # cadence and timeout/break semantics below unchanged. Reads hit the + # pollInterval=1 register cache; rxCount stays a liveness loop condition only. + SAMPLE_PERIOD = 0.5 + traj = [] + loopStart = time.monotonic() + + def _sample(): + traj.append(( + time.monotonic() - loopStart, # t_rel (s) + dcqcn.Rc.get(), # Rc (Byte/s) + dcqcn.Rt.get(), # Rt (Byte/s) + dcqcn.CnpCounter.get(), # CnpCounter (count) + dma.MonBandwidth.get(), # MonBandwidth (Gb/s) + )) + + _sample() # seed: >=1 sample even on an instant timeout + nextSample = loopStart + SAMPLE_PERIOD + + deadline = time.monotonic() + args.timeout + while root.PrbsRx.rxCount.get() < args.target: + if time.monotonic() > deadline: + print(f"WARNING: timed out after {args.timeout}s waiting for " + f"rxCount >= {args.target}", file=sys.stderr) + break # fall through to the assert; do NOT raise + if time.monotonic() >= nextSample: + _sample() + nextSample += SAMPLE_PERIOD + time.sleep(0.05) # poll interval, NOT a completion sleep + + # ---------------------------------------------------------------- + # Stop the stream (disarm dispatch first, then quiesce the PRBS source) + # ---------------------------------------------------------------- + dma.DispatchEnable.set(False) + prbs.TxEn.set(False) + + # ---------------------------------------------------------------- + # Liveness record (PRBS host counters). For the throttled run rxErrors is + # the integrity gate; for line-rate runs checkPayload is off so rxErrors is + # host-stack noise and rxCount stays a liveness precondition (frames flowed). + # ---------------------------------------------------------------- + errs = root.PrbsRx.rxErrors.get() + rxCount = root.PrbsRx.rxCount.get() + success = dma.SuccessCounter.get() + # FW diagnostic counters — read once so a single run classifies a failure: + # * success==0 with rxCount>0 -> host received frames but FW saw no ACK/CQE + # completions (ACK/return path stalled, e.g. a wedged/overrun softRoCE QP). + # * dmaReads >> success -> SENDs being retransmitted (RNR-stall: the + # responder cannot keep its recv queue armed; ~2x is the retransmit tell). + # * unsuccess>0 -> completions with non-zero WC status + # (timeout-retry exhaustion -> QP error; needs SoftReset/reconnect). + # * oversize>0 -> a SEND exceeded the FW per-SEND cap and was + # dropped (framing / PMTU mismatch), never a flow-control issue. + unsuccess = dma.UnsuccessCounter.get() + dmaReads = dma.DmaReadCount.get() + oversize = dma.OversizeCount.get() + + # ---------------------------------------------------------------- + # FW-telemetry verdict: for line-rate runs MonBandwidth (FW egress) is the + # throughput source of truth, NOT rxErrors. Numeric bands in Byte/s (Rc) and + # Gb/s (Mon). + # ---------------------------------------------------------------- + LINE_RATE_BPS = 1250000000 # 10 Gb/s in Byte/s — Rc pinned here under bypass + + # If the run timed out instantly the loop body never sampled past the seed; + # `traj` always holds >=1 entry (seeded at loop entry), but guard anyway by + # taking one live snapshot so the summary never indexes an empty list. + if not traj: + _sample() + + rc_traj = [s[1] for s in traj] + cnp_traj = [s[3] for s in traj] + mon_traj = [s[4] for s in traj] + rc_start = rc_traj[0] + rc_min = min(rc_traj) + rc_max = max(rc_traj) + rc_last = rc_traj[-1] + cnp_peak = max(cnp_traj) + mon_steady = mon_traj[-1] # last MonBandwidth Gb/s sample (steady state) + + bps2gbps = lambda b: b * 8.0 / 1.0e9 # same conversion MonBandwidth uses + + # ---------------------------------------------------------------- + # Verdict — selected by run type: + # + # * Throttled / default (--trigRate > 0, checkPayload=True): per-frame + # integrity test — PASS = zero PRBS errors AND target frames received. + # rxErrors is the source of truth; the FW-telemetry block is informational. + # + # * Line-rate (--trigRate <= 0, checkPayload off): PrbsRx is blinded, so the + # verdict gates on FW egress MonBandwidth with rxCount>=target as a liveness + # precondition. --p2p additionally requires Rc pinned at LINE_RATE; the + # baseline branch reports Rc/CnpCounter for collapse analysis but does not + # gate on them (no switch to CE-mark traffic on a switchless bench, so DCQCN + # sees no CNPs and baseline holds line rate). + # ---------------------------------------------------------------- + live = (rxCount >= args.target) + + if args.trigRate > 0: + # Throttled / default: per-frame-integrity verdict. + gate_band = "throttled per-frame integrity: rxErrors==0 AND rxCount>=target" + passed = (errs == 0) and live + elif args.p2p: + # Line-rate --p2p PASS: DCQCN bypassed -> Rc pinned at LINE_RATE, FW egress + # near line rate (margin under 9.7 for RoCEv2/UDP/IP/ETH overhead). This PASS + # demonstrates order-independence as a consequence — once DCQCN cannot + # throttle (Rc=LINE_RATE), throughput no longer depends on CNP/arm timing. + gate_band = "line-rate p2p PASS: MonBandwidth>9.0 Gb/s AND Rc==LINE_RATE" + passed = live and (mon_steady > 9.0) and (rc_last == LINE_RATE_BPS) + else: + # Line-rate baseline measurement: PASS = sustained near-line-rate FW egress. + # Rc/CnpCounter are reported in the FW-telemetry block for collapse analysis + # but are NOT gated (the collapse stimulus is bench-topology dependent). + gate_band = "line-rate baseline: MonBandwidth>9.0 Gb/s (FW egress)" + passed = live and (mon_steady > 9.0) + + if args.trigRate > 0: + run_mode = "throttled" + elif args.p2p: + run_mode = "--p2p (line-rate)" + else: + run_mode = "baseline (line-rate)" + + print( + f"--- PRBS result ---\n" + f" PrbsRx.rxErrors : {errs}\n" + f" PrbsRx.rxCount : {rxCount} (target {args.target})\n" + f" Dma.SuccessCounter : {success}\n" + f" Dma.UnsuccessCounter : {unsuccess}\n" + f" Dma.DmaReadCount : {dmaReads}\n" + f" Dma.OversizeCount : {oversize}\n" + f" RESULT : {'PASS' if passed else 'FAIL'}\n" + f"-------------------" + ) + + print( + f"--- FW telemetry ---\n" + f" run mode : {run_mode}\n" + f" trajectory samples : {len(traj)} (cadence {SAMPLE_PERIOD}s)\n" + f" Dcqcn.Rc start : {rc_start} B/s ({bps2gbps(rc_start):0.3f} Gb/s)\n" + f" Dcqcn.Rc min : {rc_min} B/s ({bps2gbps(rc_min):0.3f} Gb/s)\n" + f" Dcqcn.Rc max : {rc_max} B/s ({bps2gbps(rc_max):0.3f} Gb/s)\n" + f" Dcqcn.Rc last : {rc_last} B/s ({bps2gbps(rc_last):0.3f} Gb/s)\n" + f" Dcqcn.CnpCounter peak : {cnp_peak}\n" + f" Rdma.MonBandwidth : {mon_steady:0.3f} Gb/s (FW egress, steady)\n" + f" gate band : {gate_band}\n" + f" VERDICT : {'PASS' if passed else 'FAIL'}\n" + f"-------------------" + ) + + ###################### + # Development PyDM GUI + ###################### + if (args.guiType == 'PyDM'): + # Re-arm the stream so the operator sees rxCount climbing live; toggle + # TxEn / DispatchEnable in the GUI to start/stop. Flow control stays + # native FW<->NIC (RNR) — no credit feeder to restart. + dma.ResetCounters() # zero FW SuccessCounter/UnsuccessCounter + root.CountReset() # zero host rxErrors/rxCount/rxBytes + dma.DispatchEnable.set(True) + prbs.TxEn.set(True) + # Re-zero right before the GUI opens (the re-arm above streamed frames + # during setup). root.CountReset() cascades to the FW counters too. + root.CountReset() + pyrogue.pydm.runPyDM( + serverList = root.zmqServer.address, + sizeX = 800, + sizeY = 800, + ) + # Propagate the pass/fail code once the operator closes the GUI, so + # the default path never returns exit 0 on a FAIL run. + sys.exit(0 if passed else 1) + + ################# + # No GUI — headless, exit with the pass/fail code + ################# + elif (args.guiType == 'None'): + sys.exit(0 if passed else 1) + + #################### + # Undefined GUI type + #################### + else: + raise ValueError("Invalid GUI type (%s)" % (args.guiType)) + + ################################################################# diff --git a/software/scripts/updateBootProm.py b/software/scripts/updateBootProm.py index 6048653..5b5f28c 100644 --- a/software/scripts/updateBootProm.py +++ b/software/scripts/updateBootProm.py @@ -123,7 +123,7 @@ if (progDone): print('\nReloading FPGA firmware from PROM ....') AxiVersion.FpgaReload() - time.sleep(5) + time.sleep(10) print('\nReloading FPGA done') print ( '###################################################') diff --git a/software/setup_env_slac.sh b/software/setup_env_slac.sh index 6780876..015dedd 100644 --- a/software/setup_env_slac.sh +++ b/software/setup_env_slac.sh @@ -6,4 +6,4 @@ source /sdf/group/faders/users/$USER/miniforge3/etc/profile.d/conda.sh ################################## # Activate Rogue conda Environment ################################## -conda activate rogue_v6.13.0 +conda activate rogue_v6.15.0