1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653 |
- /*
- * Copyright (C) 2013-2015 Mellanox Technologies Ltd.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
- FILE_LICENCE ( GPL2_OR_LATER );
-
- #include <errno.h>
- #include <strings.h>
- #include <ipxe/malloc.h>
- #include <ipxe/umalloc.h>
- #include <ipxe/infiniband.h>
- #include <ipxe/ib_smc.h>
- #include <ipxe/iobuf.h>
- #include <ipxe/netdevice.h>
- #include "flexboot_nodnic.h"
- #include <ipxe/ethernet.h>
- #include <ipxe/if_ether.h>
- #include <usr/ifmgmt.h>
- #include <ipxe/in.h>
- #include <byteswap.h>
- #include "mlx_utils/include/public/mlx_pci_gw.h"
- #include <config/general.h>
- #include <ipxe/ipoib.h>
- #include "mlx_nodnic/include/mlx_port.h"
- #include "nodnic_shomron_prm.h"
- #include "golan.h"
- #include "mlx_utils/include/public/mlx_bail.h"
- #include "mlx_utils/mlx_lib/mlx_link_speed/mlx_link_speed.h"
-
-
- #define DEVICE_IS_CIB( device ) ( device == 0x1011 )
-
- /******************************************************************************/
- /************* Very simple memory management for umalloced pages **************/
- /******* Temporary solution until full memory management is implemented *******/
- /******************************************************************************/
-
- struct golan_page {
- struct list_head list;
- userptr_t addr;
- };
-
- static void golan_free_fw_areas ( struct golan *golan ) {
- int i;
-
- for (i = 0; i < GOLAN_FW_AREAS_NUM; i++) {
- if ( golan->fw_areas[i].area ) {
- ufree ( golan->fw_areas[i].area );
- golan->fw_areas[i].area = UNULL;
- }
- }
- }
-
- static int golan_init_fw_areas ( struct golan *golan ) {
- int rc = 0, i = 0;
-
- if ( ! golan ) {
- rc = -EINVAL;
- goto err_golan_init_fw_areas_bad_param;
- }
-
- for (i = 0; i < GOLAN_FW_AREAS_NUM; i++)
- golan->fw_areas[i].area = UNULL;
-
- return rc;
-
- err_golan_init_fw_areas_bad_param:
- return rc;
- }
-
- /******************************************************************************/
-
- const char *golan_qp_state_as_string[] = {
- "RESET",
- "INIT",
- "RTR",
- "RTS",
- "SQD",
- "SQE",
- "ERR"
- };
-
- static inline int golan_check_rc_and_cmd_status ( struct golan_cmd_layout *cmd, int rc ) {
- struct golan_outbox_hdr *out_hdr = ( struct golan_outbox_hdr * ) ( cmd->out );
- if ( rc == -EBUSY ) {
- DBG ( "HCA is busy (rc = -EBUSY)\n" );
- return rc;
- } else if ( out_hdr->status ) {
- DBG("%s status = 0x%x - syndrom = 0x%x\n", __FUNCTION__,
- out_hdr->status, be32_to_cpu(out_hdr->syndrome));
- return out_hdr->status;
- }
- return 0;
- }
-
- #define GOLAN_CHECK_RC_AND_CMD_STATUS(_lable) \
- do { \
- if ( ( rc = golan_check_rc_and_cmd_status ( cmd, rc ) ) ) \
- goto _lable; \
- } while (0)
-
- #define GOLAN_PRINT_RC_AND_CMD_STATUS golan_check_rc_and_cmd_status ( cmd, rc )
-
-
- struct mbox {
- union {
- struct golan_cmd_prot_block mblock;
- u8 data[MAILBOX_STRIDE];
- __be64 qdata[MAILBOX_STRIDE >> 3];
- };
- };
-
- static inline uint32_t ilog2(uint32_t mem)
- {
- return ( fls ( mem ) - 1 );
- }
-
- #define CTRL_SIG_SZ (sizeof(mailbox->mblock) - sizeof(mailbox->mblock.bdata) - 2)
-
- static inline u8 xor8_buf(void *buf, int len)
- {
- u8 sum = 0;
- int i;
- u8 *ptr = buf;
-
- for (i = 0; i < len; ++i)
- sum ^= ptr[i];
-
- return sum;
- }
-
- static inline const char *cmd_status_str(u8 status)
- {
- switch (status) {
- case 0x0: return "OK";
- case 0x1: return "internal error";
- case 0x2: return "bad operation";
- case 0x3: return "bad parameter";
- case 0x4: return "bad system state";
- case 0x5: return "bad resource";
- case 0x6: return "resource busy";
- case 0x8: return "limits exceeded";
- case 0x9: return "bad resource state";
- case 0xa: return "bad index";
- case 0xf: return "no resources";
- case 0x50: return "bad input length";
- case 0x51: return "bad output length";
- case 0x10: return "bad QP state";
- case 0x30: return "bad packet (discarded)";
- case 0x40: return "bad size too many outstanding CQEs";
- case 0xff: return "Command Timed Out";
- default: return "unknown status";
- }
- }
-
- static inline uint16_t fw_rev_maj(struct golan *golan)
- {
- return be32_to_cpu(readl(&golan->iseg->fw_rev)) & 0xffff;
- }
-
- static inline u16 fw_rev_min(struct golan *golan)
- {
- return be32_to_cpu(readl(&golan->iseg->fw_rev)) >> 16;
- }
-
- static inline u16 fw_rev_sub(struct golan *golan)
- {
- return be32_to_cpu(readl(&golan->iseg->cmdif_rev_fw_sub)) & 0xffff;
- }
-
- static inline u16 cmdif_rev(struct golan *golan)
- {
- return be32_to_cpu(readl(&golan->iseg->cmdif_rev_fw_sub)) >> 16;
- }
-
-
- static inline struct golan_cmd_layout *get_cmd( struct golan *golan, int idx )
- {
- return golan->cmd.addr + (idx << golan->cmd.log_stride);
- }
-
- static inline void golan_calc_sig(struct golan *golan, uint32_t cmd_idx,
- uint32_t inbox_idx, uint32_t outbox_idx)
- {
- struct golan_cmd_layout *cmd = get_cmd(golan, cmd_idx);
- struct mbox *mailbox = NULL;
-
- if (inbox_idx != NO_MBOX) {
- mailbox = GET_INBOX(golan, inbox_idx);
- mailbox->mblock.token = cmd->token;
- mailbox->mblock.ctrl_sig = ~xor8_buf(mailbox->mblock.rsvd0,
- CTRL_SIG_SZ);
- }
- if (outbox_idx != NO_MBOX) {
- mailbox = GET_OUTBOX(golan, outbox_idx);
- mailbox->mblock.token = cmd->token;
- mailbox->mblock.ctrl_sig = ~xor8_buf(mailbox->mblock.rsvd0,
- CTRL_SIG_SZ);
- }
- cmd->sig = ~xor8_buf(cmd, sizeof(*cmd));
- }
-
- static inline void show_out_status(uint32_t *out)
- {
- DBG("%x\n", be32_to_cpu(out[0]));
- DBG("%x\n", be32_to_cpu(out[1]));
- DBG("%x\n", be32_to_cpu(out[2]));
- DBG("%x\n", be32_to_cpu(out[3]));
- }
- /**
- * Check if CMD has finished.
- */
- static inline uint32_t is_command_finished( struct golan *golan, int idx)
- {
- wmb();
- return !(get_cmd( golan , idx )->status_own & CMD_OWNER_HW);
- }
-
- /**
- * Wait for Golan command completion
- *
- * @v golan Golan device
- * @ret rc Return status code
- */
- static inline int golan_cmd_wait(struct golan *golan, int idx, const char *command)
- {
- unsigned int wait;
- int rc = -EBUSY;
-
- for ( wait = GOLAN_HCR_MAX_WAIT_MS ; wait ; --wait ) {
- if (is_command_finished(golan, idx)) {
- rc = CMD_STATUS(golan, idx);
- rmb();
- break;
- } else {
- mdelay ( 1 );
- }
- }
- if (rc) {
- DBGC (golan ,"[%s]RC is %s[%x]\n", command, cmd_status_str(rc), rc);
- }
-
- golan->cmd_bm &= ~(1 << idx);
- return rc;
- }
-
- /**
- * Notify the HW that commands are ready
- */
- static inline void send_command(struct golan *golan)
- {
- wmb(); //Make sure the command is visible in "memory".
- writel(cpu_to_be32(golan->cmd_bm) , &golan->iseg->cmd_dbell);
- }
-
- static inline int send_command_and_wait(struct golan *golan, uint32_t cmd_idx,
- uint32_t inbox_idx, uint32_t outbox_idx, const char *command)
- {
- golan_calc_sig(golan, cmd_idx, inbox_idx, outbox_idx);
- send_command(golan);
- return golan_cmd_wait(golan, cmd_idx, command);
- }
-
- /**
- * Prepare a FW command,
- * In - comamnd idx (Must be valid)
- * writes the command parameters.
- */
- static inline struct golan_cmd_layout *write_cmd(struct golan *golan, int idx,
- uint16_t opcode, uint16_t opmod,
- uint16_t inbox_idx,
- uint16_t outbox_idx, uint16_t inlen,
- uint16_t outlen)
- {
- struct golan_cmd_layout *cmd = get_cmd(golan , idx);
- struct golan_inbox_hdr *hdr = (struct golan_inbox_hdr *)cmd->in;
- static uint8_t token;
-
- memset(cmd, 0, sizeof(*cmd));
-
- cmd->type = GOLAN_PCI_CMD_XPORT;
- cmd->status_own = CMD_OWNER_HW;
- cmd->outlen = cpu_to_be32(outlen);
- cmd->inlen = cpu_to_be32(inlen);
- hdr->opcode = cpu_to_be16(opcode);
- hdr->opmod = cpu_to_be16(opmod);
-
- if (inbox_idx != NO_MBOX) {
- memset(GET_INBOX(golan, inbox_idx), 0, MAILBOX_SIZE);
- cmd->in_ptr = VIRT_2_BE64_BUS(GET_INBOX(golan, inbox_idx));
- cmd->token = ++token;
- }
- if (outbox_idx != NO_MBOX) {
- memset(GET_OUTBOX(golan, outbox_idx), 0, MAILBOX_SIZE);
- cmd->out_ptr = VIRT_2_BE64_BUS(GET_OUTBOX(golan, outbox_idx));
- }
-
- golan->cmd_bm |= 1 << idx;
-
- assert ( cmd != NULL );
- return cmd;
- }
-
- static inline int golan_core_enable_hca(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- int rc = 0;
-
- DBGC(golan, "%s\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_ENABLE_HCA, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_enable_hca_mbox_in),
- sizeof(struct golan_enable_hca_mbox_out));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- return rc;
- }
-
- static inline void golan_disable_hca(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DISABLE_HCA, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_disable_hca_mbox_in),
- sizeof(struct golan_disable_hca_mbox_out));
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- }
-
- static inline int golan_set_hca_cap(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- int rc;
-
- DBGC(golan, "%s\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_SET_HCA_CAP, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_cmd_set_hca_cap_mbox_in),
- sizeof(struct golan_cmd_set_hca_cap_mbox_out));
-
- golan->caps.flags &= ~GOLAN_DEV_CAP_FLAG_CMDIF_CSUM;
- DBGC( golan , "%s caps.uar_sz = %d\n", __FUNCTION__, golan->caps.uar_sz);
- DBGC( golan , "%s caps.log_pg_sz = %d\n", __FUNCTION__, golan->caps.log_pg_sz);
- DBGC( golan , "%s caps.log_uar_sz = %d\n", __FUNCTION__, be32_to_cpu(golan->caps.uar_page_sz));
- golan->caps.uar_page_sz = 0;
- golan->caps.log_max_qp = GOLAN_LOG_MAX_QP;
-
- memcpy(((struct golan_hca_cap *)GET_INBOX(golan, GEN_MBOX)),
- &(golan->caps),
- sizeof(struct golan_hca_cap));
-
- //if command failed we should reset the caps in golan->caps
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- return rc;
- }
-
- static inline int golan_qry_hca_cap(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- int rc = 0;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_QUERY_HCA_CAP, 0x1,
- NO_MBOX, GEN_MBOX,
- sizeof(struct golan_cmd_query_hca_cap_mbox_in),
- sizeof(struct golan_cmd_query_hca_cap_mbox_out));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, GEN_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_query_hca_cap );
-
- memcpy(&(golan->caps),
- ((struct golan_hca_cap *)GET_OUTBOX(golan, GEN_MBOX)),
- sizeof(struct golan_hca_cap));
- err_query_hca_cap:
- return rc;
- }
-
- static inline int golan_take_pages ( struct golan *golan, uint32_t pages, __be16 func_id ) {
- uint32_t out_num_entries = 0;
- int size_ibox = 0;
- int size_obox = 0;
- int rc = 0;
-
- DBGC(golan, "%s\n", __FUNCTION__);
-
- while ( pages > 0 ) {
- uint32_t pas_num = min(pages, MAX_PASE_MBOX);
- struct golan_cmd_layout *cmd;
- struct golan_manage_pages_inbox *in;
-
- size_ibox = sizeof(struct golan_manage_pages_inbox) + (pas_num * GOLAN_PAS_SIZE);
- size_obox = sizeof(struct golan_manage_pages_outbox) + (pas_num * GOLAN_PAS_SIZE);
-
- cmd = write_cmd(golan, MEM_CMD_IDX, GOLAN_CMD_OP_MANAGE_PAGES, GOLAN_PAGES_TAKE,
- MEM_MBOX, MEM_MBOX,
- size_ibox,
- size_obox);
-
- in = (struct golan_manage_pages_inbox *)cmd->in; /* Warning (WE CANT USE THE LAST 2 FIELDS) */
-
- in->func_id = func_id; /* Already BE */
- in->num_entries = cpu_to_be32(pas_num);
-
- if ( ( rc = send_command_and_wait(golan, MEM_CMD_IDX, MEM_MBOX, MEM_MBOX, __FUNCTION__) ) == 0 ) {
- out_num_entries = be32_to_cpu(((struct golan_manage_pages_outbox *)(cmd->out))->num_entries);
- } else {
- if ( rc == -EBUSY ) {
- DBGC (golan ,"HCA is busy (rc = -EBUSY)\n" );
- } else {
- DBGC (golan ,"%s: rc =0x%x[%s]<%x> syn 0x%x[0x%x] for %d pages\n",
- __FUNCTION__, rc, cmd_status_str(rc),
- CMD_SYND(golan, MEM_CMD_IDX),
- get_cmd( golan , MEM_CMD_IDX )->status_own,
- be32_to_cpu(CMD_SYND(golan, MEM_CMD_IDX)), pas_num);
- }
- return rc;
- }
-
- pages -= out_num_entries;
- }
- DBGC( golan , "%s Pages handled\n", __FUNCTION__);
- return rc;
- }
-
- static inline int golan_provide_pages ( struct golan *golan , uint32_t pages
- , __be16 func_id,struct golan_firmware_area *fw_area) {
- struct mbox *mailbox;
- int size_ibox = 0;
- int size_obox = 0;
- int rc = 0;
- userptr_t next_page_addr = UNULL;
-
- DBGC(golan, "%s\n", __FUNCTION__);
- if ( ! fw_area->area ) {
- fw_area->area = umalloc ( GOLAN_PAGE_SIZE * pages );
- if ( fw_area->area == UNULL ) {
- rc = -ENOMEM;
- DBGC (golan ,"Failed to allocated %d pages \n",pages);
- goto err_golan_alloc_fw_area;
- }
- fw_area->npages = pages;
- }
- assert ( fw_area->npages == pages );
- next_page_addr = fw_area->area;
- while ( pages > 0 ) {
- uint32_t pas_num = min(pages, MAX_PASE_MBOX);
- unsigned i, j;
- struct golan_cmd_layout *cmd;
- struct golan_manage_pages_inbox *in;
- userptr_t addr = 0;
-
- mailbox = GET_INBOX(golan, MEM_MBOX);
- size_ibox = sizeof(struct golan_manage_pages_inbox) + (pas_num * GOLAN_PAS_SIZE);
- size_obox = sizeof(struct golan_manage_pages_outbox) + (pas_num * GOLAN_PAS_SIZE);
-
- cmd = write_cmd(golan, MEM_CMD_IDX, GOLAN_CMD_OP_MANAGE_PAGES, GOLAN_PAGES_GIVE,
- MEM_MBOX, MEM_MBOX,
- size_ibox,
- size_obox);
-
- in = (struct golan_manage_pages_inbox *)cmd->in; /* Warning (WE CANT USE THE LAST 2 FIELDS) */
-
- in->func_id = func_id; /* Already BE */
- in->num_entries = cpu_to_be32(pas_num);
-
- for ( i = 0 , j = MANAGE_PAGES_PSA_OFFSET; i < pas_num; ++i ,++j,
- next_page_addr += GOLAN_PAGE_SIZE ) {
- addr = next_page_addr;
- if (GOLAN_PAGE_MASK & user_to_phys(addr, 0)) {
- DBGC (golan ,"Addr not Page alligned [%lx %lx]\n", user_to_phys(addr, 0), addr);
- }
- mailbox->mblock.data[j] = USR_2_BE64_BUS(addr);
- }
-
- if ( ( rc = send_command_and_wait(golan, MEM_CMD_IDX, MEM_MBOX, MEM_MBOX, __FUNCTION__) ) == 0 ) {
- pages -= pas_num;
- golan->total_dma_pages += pas_num;
- } else {
- if ( rc == -EBUSY ) {
- DBGC (golan ,"HCA is busy (rc = -EBUSY)\n" );
- } else {
- DBGC (golan ,"%s: rc =0x%x[%s]<%x> syn 0x%x[0x%x] for %d pages\n",
- __FUNCTION__, rc, cmd_status_str(rc),
- CMD_SYND(golan, MEM_CMD_IDX),
- get_cmd( golan , MEM_CMD_IDX )->status_own,
- be32_to_cpu(CMD_SYND(golan, MEM_CMD_IDX)), pas_num);
- }
- goto err_send_command;
- }
- }
- DBGC( golan , "%s Pages handled\n", __FUNCTION__);
- return 0;
-
- err_send_command:
- err_golan_alloc_fw_area:
- /* Go over In box and free pages */
- /* Send Error to FW */
- /* What is next - Disable HCA? */
- DBGC (golan ,"%s Failed (rc = 0x%x)\n", __FUNCTION__, rc);
- return rc;
- }
-
- static inline int golan_handle_pages(struct golan *golan,
- enum golan_qry_pages_mode qry,
- enum golan_manage_pages_mode mode)
- {
- struct golan_cmd_layout *cmd;
-
- int rc = 0;
- int32_t pages;
- uint16_t total_pages;
- __be16 func_id;
-
- DBGC(golan, "%s\n", __FUNCTION__);
-
- cmd = write_cmd(golan, MEM_CMD_IDX, GOLAN_CMD_OP_QUERY_PAGES, qry,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_query_pages_inbox),
- sizeof(struct golan_query_pages_outbox));
-
- rc = send_command_and_wait(golan, MEM_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_handle_pages_query );
-
- pages = be32_to_cpu(QRY_PAGES_OUT(golan, MEM_CMD_IDX)->num_pages);
-
- DBGC( golan , "%s pages needed: %d\n", __FUNCTION__, pages);
-
- func_id = QRY_PAGES_OUT(golan, MEM_CMD_IDX)->func_id;
-
- total_pages = (( pages >= 0 ) ? pages : ( pages * ( -1 ) ));
-
- if ( mode == GOLAN_PAGES_GIVE ) {
- rc = golan_provide_pages(golan, total_pages, func_id, & ( golan->fw_areas[qry-1] ));
- } else {
- rc = golan_take_pages(golan, golan->total_dma_pages, func_id);
- golan->total_dma_pages = 0;
- }
-
- if ( rc ) {
- DBGC (golan , "Failed to %s pages (rc = %d) - DMA pages allocated = %d\n",
- ( ( mode == GOLAN_PAGES_GIVE ) ? "give" : "take" ), rc , golan->total_dma_pages );
- return rc;
- }
-
- return 0;
-
- err_handle_pages_query:
- DBGC (golan ,"%s Qyery pages failed (rc = 0x%x)\n", __FUNCTION__, rc);
- return rc;
- }
-
- static inline int golan_set_access_reg ( struct golan *golan __attribute__ (( unused )), uint32_t reg __attribute__ (( unused )))
- {
- #if 0
- write_cmd(golan, _CMD_IDX, GOLAN_CMD_OP_QUERY_PAGES, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_reg_host_endianess),
- sizeof(struct golan_reg_host_endianess));
- in->arg = cpu_to_be32(arg);
- in->register_id = cpu_to_be16(reg_num);
- #endif
- DBGC (golan ," %s Not implemented yet\n", __FUNCTION__);
- return 0;
- }
-
- static inline void golan_cmd_uninit ( struct golan *golan )
- {
- free_dma(golan->mboxes.outbox, GOLAN_PAGE_SIZE);
- free_dma(golan->mboxes.inbox, GOLAN_PAGE_SIZE);
- free_dma(golan->cmd.addr, GOLAN_PAGE_SIZE);
- }
-
- /**
- * Initialise Golan Command Q parameters
- * -- Alocate a 4kb page for the Command Q
- * -- Read the stride and log num commands available
- * -- Write the address to cmdq_phy_addr in iseg
- * @v golan Golan device
- */
- static inline int golan_cmd_init ( struct golan *golan )
- {
- int rc = 0;
- uint32_t addr_l_sz;
-
- if (!(golan->cmd.addr = malloc_dma(GOLAN_PAGE_SIZE , GOLAN_PAGE_SIZE))) {
- rc = -ENOMEM;
- goto malloc_dma_failed;
- }
- if (!(golan->mboxes.inbox = malloc_dma(GOLAN_PAGE_SIZE , GOLAN_PAGE_SIZE))) {
- rc = -ENOMEM;
- goto malloc_dma_inbox_failed;
- }
- if (!(golan->mboxes.outbox = malloc_dma(GOLAN_PAGE_SIZE , GOLAN_PAGE_SIZE))) {
- rc = -ENOMEM;
- goto malloc_dma_outbox_failed;
- }
- addr_l_sz = be32_to_cpu(readl(&golan->iseg->cmdq_addr_l_sz));
-
- golan->cmd.log_stride = addr_l_sz & 0xf;
- golan->cmd.size = 1 << (( addr_l_sz >> 4 ) & 0xf);
-
- addr_l_sz = virt_to_bus(golan->cmd.addr);
- writel(0 /* cpu_to_be32(golan->cmd.addr) >> 32 */, &golan->iseg->cmdq_addr_h);
- writel(cpu_to_be32(addr_l_sz), &golan->iseg->cmdq_addr_l_sz);
- wmb(); //Make sure the addr is visible in "memory".
-
- addr_l_sz = be32_to_cpu(readl(&golan->iseg->cmdq_addr_l_sz));
-
- DBGC( golan , "%s Command interface was initialized\n", __FUNCTION__);
- return 0;
-
- malloc_dma_outbox_failed:
- free_dma(golan->mboxes.inbox, GOLAN_PAGE_SIZE);
- malloc_dma_inbox_failed:
- free_dma(golan->cmd.addr, GOLAN_PAGE_SIZE);
- malloc_dma_failed:
- DBGC (golan ,"%s Failed to initialize command interface (rc = 0x%x)\n",
- __FUNCTION__, rc);
- return rc;
- }
-
- static inline int golan_hca_init(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- int rc = 0;
-
- DBGC(golan, "%s\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_INIT_HCA, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_cmd_init_hca_mbox_in),
- sizeof(struct golan_cmd_init_hca_mbox_out));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- return rc;
- }
-
- static inline void golan_teardown_hca(struct golan *golan, enum golan_teardown op_mod)
- {
- struct golan_cmd_layout *cmd;
- int rc;
-
- DBGC (golan, "%s in\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_TEARDOWN_HCA, op_mod,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_cmd_teardown_hca_mbox_in),
- sizeof(struct golan_cmd_teardown_hca_mbox_out));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
-
- DBGC (golan, "%s HCA teardown compleated\n", __FUNCTION__);
- }
-
- static inline int golan_alloc_uar(struct golan *golan)
- {
- struct golan_uar *uar = &golan->uar;
- struct golan_cmd_layout *cmd;
- struct golan_alloc_uar_mbox_out *out;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_ALLOC_UAR, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_alloc_uar_mbox_in),
- sizeof(struct golan_alloc_uar_mbox_out));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_alloc_uar_cmd );
- out = (struct golan_alloc_uar_mbox_out *) ( cmd->out );
-
- uar->index = be32_to_cpu(out->uarn) & 0xffffff;
-
- uar->phys = (pci_bar_start(golan->pci, GOLAN_HCA_BAR) + (uar->index << GOLAN_PAGE_SHIFT));
- uar->virt = (void *)(ioremap(uar->phys, GOLAN_PAGE_SIZE));
-
- DBGC( golan , "%s: UAR allocated with index 0x%x\n", __FUNCTION__, uar->index);
- return 0;
-
- err_alloc_uar_cmd:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static void golan_dealloc_uar(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- uint32_t uar_index = golan->uar.index;
- int rc;
-
- DBGC (golan, "%s in\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DEALLOC_UAR, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_free_uar_mbox_in),
- sizeof(struct golan_free_uar_mbox_out));
-
- ((struct golan_free_uar_mbox_in *)(cmd->in))->uarn = cpu_to_be32(uar_index);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- golan->uar.index = 0;
-
- DBGC (golan, "%s UAR (0x%x) was destroyed\n", __FUNCTION__, uar_index);
- }
-
- static void golan_eq_update_ci(struct golan_event_queue *eq, int arm)
- {
- __be32 *addr = eq->doorbell + (arm ? 0 : 2);
- u32 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
- writel(cpu_to_be32(val) , addr);
- /* We still want ordering, just not swabbing, so add a barrier */
- wmb();
- }
-
- static int golan_create_eq(struct golan *golan)
- {
- struct golan_event_queue *eq = &golan->eq;
- struct golan_create_eq_mbox_in_data *in;
- struct golan_cmd_layout *cmd;
- struct golan_create_eq_mbox_out *out;
- int rc, i;
-
- eq->cons_index = 0;
- eq->size = GOLAN_NUM_EQES * sizeof(eq->eqes[0]);
- eq->eqes = malloc_dma ( GOLAN_PAGE_SIZE, GOLAN_PAGE_SIZE );
- if (!eq->eqes) {
- rc = -ENOMEM;
- goto err_create_eq_eqe_alloc;
- }
-
- /* Set EQEs ownership bit to HW ownership */
- for (i = 0; i < GOLAN_NUM_EQES; ++i) {
- eq->eqes[i].owner = GOLAN_EQE_HW_OWNERSHIP;
- }
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_CREATE_EQ, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_create_eq_mbox_in) + GOLAN_PAS_SIZE,
- sizeof(struct golan_create_eq_mbox_out));
-
- in = (struct golan_create_eq_mbox_in_data *)GET_INBOX(golan, GEN_MBOX);
-
- /* Fill the physical address of the page */
- in->pas[0] = VIRT_2_BE64_BUS( eq->eqes );
- in->ctx.log_sz_usr_page = cpu_to_be32((ilog2(GOLAN_NUM_EQES)) << 24 | golan->uar.index);
- DBGC( golan , "UAR idx %x (BE %x)\n", golan->uar.index, in->ctx.log_sz_usr_page);
- in->events_mask = cpu_to_be64(1 << GOLAN_EVENT_TYPE_PORT_CHANGE);
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_create_eq_cmd );
- out = (struct golan_create_eq_mbox_out *)cmd->out;
-
- eq->eqn = out->eq_number;
- eq->doorbell = ((void *)golan->uar.virt) + GOLAN_EQ_DOORBELL_OFFSET;
-
- /* EQs are created in ARMED state */
- golan_eq_update_ci(eq, GOLAN_EQ_UNARMED);
-
- DBGC( golan , "%s: Event queue created (EQN = 0x%x)\n", __FUNCTION__, eq->eqn);
- return 0;
-
- err_create_eq_cmd:
- free_dma ( eq->eqes , GOLAN_PAGE_SIZE );
- err_create_eq_eqe_alloc:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static void golan_destory_eq(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- struct golan_destroy_eq_mbox_in *in;
- uint8_t eqn = golan->eq.eqn;
- int rc;
-
- DBGC (golan, "%s in\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DESTROY_EQ, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_destroy_eq_mbox_in),
- sizeof(struct golan_destroy_eq_mbox_out));
-
- in = GOLAN_MBOX_IN ( cmd, in );
- in->eqn = eqn;
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
-
- free_dma ( golan->eq.eqes , GOLAN_PAGE_SIZE );
- golan->eq.eqn = 0;
-
- DBGC( golan, "%s Event queue (0x%x) was destroyed\n", __FUNCTION__, eqn);
- }
-
- static int golan_alloc_pd(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- struct golan_alloc_pd_mbox_out *out;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_ALLOC_PD, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_alloc_pd_mbox_in),
- sizeof(struct golan_alloc_pd_mbox_out));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_alloc_pd_cmd );
- out = (struct golan_alloc_pd_mbox_out *) ( cmd->out );
-
- golan->pdn = (be32_to_cpu(out->pdn) & 0xffffff);
- DBGC( golan , "%s: Protection domain created (PDN = 0x%x)\n", __FUNCTION__,
- golan->pdn);
- return 0;
-
- err_alloc_pd_cmd:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static void golan_dealloc_pd(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- uint32_t pdn = golan->pdn;
- int rc;
-
- DBGC (golan,"%s in\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DEALLOC_PD, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_alloc_pd_mbox_in),
- sizeof(struct golan_alloc_pd_mbox_out));
-
- ((struct golan_dealloc_pd_mbox_in *)(cmd->in))->pdn = cpu_to_be32(pdn);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- golan->pdn = 0;
-
- DBGC (golan ,"%s Protection domain (0x%x) was destroyed\n", __FUNCTION__, pdn);
- }
-
- static int golan_create_mkey(struct golan *golan)
- {
- struct golan_create_mkey_mbox_in_data *in;
- struct golan_cmd_layout *cmd;
- struct golan_create_mkey_mbox_out *out;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_CREATE_MKEY, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_create_mkey_mbox_in),
- sizeof(struct golan_create_mkey_mbox_out));
-
- in = (struct golan_create_mkey_mbox_in_data *)GET_INBOX(golan, GEN_MBOX);
-
- in->seg.flags = GOLAN_IB_ACCESS_LOCAL_WRITE | GOLAN_IB_ACCESS_LOCAL_READ;
- in->seg.flags_pd = cpu_to_be32(golan->pdn | GOLAN_MKEY_LEN64);
- in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << GOLAN_CREATE_MKEY_SEG_QPN_BIT);
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_create_mkey_cmd );
- out = (struct golan_create_mkey_mbox_out *) ( cmd->out );
-
- golan->mkey = ((be32_to_cpu(out->mkey) & 0xffffff) << 8);
- DBGC( golan , "%s: Got DMA Key for local access read/write (MKEY = 0x%x)\n",
- __FUNCTION__, golan->mkey);
- return 0;
- err_create_mkey_cmd:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static void golan_destroy_mkey(struct golan *golan)
- {
- struct golan_cmd_layout *cmd;
- u32 mkey = golan->mkey;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DESTROY_MKEY, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_destroy_mkey_mbox_in),
- sizeof(struct golan_destroy_mkey_mbox_out));
- ((struct golan_destroy_mkey_mbox_in *)(cmd->in))->mkey = cpu_to_be32(mkey >> 8);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- golan->mkey = 0;
-
- DBGC( golan , "%s DMA Key (0x%x) for local access write was destroyed\n"
- , __FUNCTION__, mkey);
- }
-
-
- /**
- * Initialise Golan PCI parameters
- *
- * @v golan Golan device
- */
- static inline void golan_pci_init(struct golan *golan)
- {
- struct pci_device *pci = golan->pci;
-
- /* Fix up PCI device */
- adjust_pci_device ( pci );
-
- /* Get HCA BAR */
- golan->iseg = ioremap ( pci_bar_start ( pci, GOLAN_HCA_BAR),
- GOLAN_PCI_CONFIG_BAR_SIZE );
- }
-
- static inline struct golan *golan_alloc()
- {
- void *golan = zalloc(sizeof(struct golan));
- if ( !golan )
- goto err_zalloc;
-
- return golan;
-
- err_zalloc:
- return NULL;
- }
-
- /**
- * Create completion queue
- *
- * @v ibdev Infiniband device
- * @v cq Completion queue
- * @ret rc Return status code
- */
- static int golan_create_cq(struct ib_device *ibdev,
- struct ib_completion_queue *cq)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_completion_queue *golan_cq;
- struct golan_cmd_layout *cmd;
- struct golan_create_cq_mbox_in_data *in;
- struct golan_create_cq_mbox_out *out;
- int rc;
- unsigned int i;
-
- golan_cq = zalloc(sizeof(*golan_cq));
- if (!golan_cq) {
- rc = -ENOMEM;
- goto err_create_cq;
- }
- golan_cq->size = sizeof(golan_cq->cqes[0]) * cq->num_cqes;
- golan_cq->doorbell_record = malloc_dma(GOLAN_CQ_DB_RECORD_SIZE,
- GOLAN_CQ_DB_RECORD_SIZE);
- if (!golan_cq->doorbell_record) {
- rc = -ENOMEM;
- goto err_create_cq_db_alloc;
- }
-
- golan_cq->cqes = malloc_dma ( GOLAN_PAGE_SIZE, GOLAN_PAGE_SIZE );
- if (!golan_cq->cqes) {
- rc = -ENOMEM;
- goto err_create_cq_cqe_alloc;
- }
-
- /* Set CQEs ownership bit to HW ownership */
- for (i = 0; i < cq->num_cqes; ++i) {
- golan_cq->cqes[i].op_own = ((GOLAN_CQE_OPCODE_NOT_VALID <<
- GOLAN_CQE_OPCODE_BIT) |
- GOLAN_CQE_HW_OWNERSHIP);
- }
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_CREATE_CQ, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_create_cq_mbox_in) + GOLAN_PAS_SIZE,
- sizeof(struct golan_create_cq_mbox_out));
-
- in = (struct golan_create_cq_mbox_in_data *)GET_INBOX(golan, GEN_MBOX);
-
- /* Fill the physical address of the page */
- in->pas[0] = VIRT_2_BE64_BUS( golan_cq->cqes );
- in->ctx.cqe_sz_flags = GOLAN_CQE_SIZE_64 << 5;
- in->ctx.log_sz_usr_page = cpu_to_be32(((ilog2(cq->num_cqes)) << 24) | golan->uar.index);
- in->ctx.c_eqn = cpu_to_be16(golan->eq.eqn);
- in->ctx.db_record_addr = VIRT_2_BE64_BUS(golan_cq->doorbell_record);
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_create_cq_cmd );
- out = (struct golan_create_cq_mbox_out *) ( cmd->out );
-
- cq->cqn = (be32_to_cpu(out->cqn) & 0xffffff);
-
- ib_cq_set_drvdata(cq, golan_cq);
-
- DBGC( golan , "%s CQ created successfully (CQN = 0x%lx)\n", __FUNCTION__, cq->cqn);
- return 0;
-
- err_create_cq_cmd:
- free_dma( golan_cq->cqes , GOLAN_PAGE_SIZE );
- err_create_cq_cqe_alloc:
- free_dma(golan_cq->doorbell_record, GOLAN_CQ_DB_RECORD_SIZE);
- err_create_cq_db_alloc:
- free ( golan_cq );
- err_create_cq:
- DBGC (golan ,"%s out rc = 0x%x\n", __FUNCTION__, rc);
- return rc;
- }
-
- /**
- * Destroy completion queue
- *
- * @v ibdev Infiniband device
- * @v cq Completion queue
- */
- static void golan_destroy_cq(struct ib_device *ibdev,
- struct ib_completion_queue *cq)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_completion_queue *golan_cq = ib_cq_get_drvdata(cq);
- struct golan_cmd_layout *cmd;
- uint32_t cqn = cq->cqn;
- int rc;
-
- DBGC (golan, "%s in\n", __FUNCTION__);
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DESTROY_CQ, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_destroy_cq_mbox_in),
- sizeof(struct golan_destroy_cq_mbox_out));
- ((struct golan_destroy_cq_mbox_in *)(cmd->in))->cqn = cpu_to_be32(cqn);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- cq->cqn = 0;
-
- ib_cq_set_drvdata(cq, NULL);
- free_dma ( golan_cq->cqes , GOLAN_PAGE_SIZE );
- free_dma(golan_cq->doorbell_record, GOLAN_CQ_DB_RECORD_SIZE);
- free(golan_cq);
-
- DBGC (golan, "%s CQ number 0x%x was destroyed\n", __FUNCTION__, cqn);
- }
-
- static void golan_cq_clean(struct ib_completion_queue *cq)
- {
- ib_poll_cq(cq->ibdev, cq);
- }
-
- static int golan_qp_type_to_st(enum ib_queue_pair_type type)
- {
- int qpt = type;
-
- switch (qpt) {
- case IB_QPT_RC:
- return GOLAN_QP_ST_RC;
- case IB_QPT_UD:
- return GOLAN_QP_ST_UD;
- case IB_QPT_SMI:
- return GOLAN_QP_ST_QP0;
- case IB_QPT_GSI:
- return GOLAN_QP_ST_QP1;
- case IB_QPT_ETH:
- default:
- return -EINVAL;
- }
- }
- #if 0
- static int golan_is_special_qp(enum ib_queue_pair_type type)
- {
- return (type == IB_QPT_GSI || type == IB_QPT_SMI);
- }
- #endif
- static int golan_create_qp_aux(struct ib_device *ibdev,
- struct ib_queue_pair *qp,
- int *qpn)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_queue_pair *golan_qp;
- struct golan_create_qp_mbox_in_data *in;
- struct golan_cmd_layout *cmd;
- struct golan_wqe_data_seg *data;
- struct golan_create_qp_mbox_out *out;
- uint32_t wqe_size_in_bytes;
- uint32_t max_qp_size_in_wqes;
- unsigned int i;
- int rc;
-
- golan_qp = zalloc(sizeof(*golan_qp));
- if (!golan_qp) {
- rc = -ENOMEM;
- goto err_create_qp;
- }
-
- if ( ( qp->type == IB_QPT_SMI ) || ( qp->type == IB_QPT_GSI ) ||
- ( qp->type == IB_QPT_UD ) ) {
- golan_qp->rq.grh_size = ( qp->recv.num_wqes *
- sizeof ( golan_qp->rq.grh[0] ));
- }
-
- /* Calculate receive queue size */
- golan_qp->rq.size = qp->recv.num_wqes * GOLAN_RECV_WQE_SIZE;
- if (GOLAN_RECV_WQE_SIZE > be16_to_cpu(golan->caps.max_wqe_sz_rq)) {
- DBGC (golan ,"%s receive wqe size [%zd] > max wqe size [%d]\n", __FUNCTION__,
- GOLAN_RECV_WQE_SIZE, be16_to_cpu(golan->caps.max_wqe_sz_rq));
- rc = -EINVAL;
- goto err_create_qp_rq_size;
- }
-
- wqe_size_in_bytes = sizeof(golan_qp->sq.wqes[0]);
- /* Calculate send queue size */
- if (wqe_size_in_bytes > be16_to_cpu(golan->caps.max_wqe_sz_sq)) {
- DBGC (golan ,"%s send WQE size [%d] > max WQE size [%d]\n", __FUNCTION__,
- wqe_size_in_bytes,
- be16_to_cpu(golan->caps.max_wqe_sz_sq));
- rc = -EINVAL;
- goto err_create_qp_sq_wqe_size;
- }
- golan_qp->sq.size = (qp->send.num_wqes * wqe_size_in_bytes);
- max_qp_size_in_wqes = (1 << ((uint32_t)(golan->caps.log_max_qp_sz)));
- if (qp->send.num_wqes > max_qp_size_in_wqes) {
- DBGC (golan ,"%s send wq size [%d] > max wq size [%d]\n", __FUNCTION__,
- golan_qp->sq.size, max_qp_size_in_wqes);
- rc = -EINVAL;
- goto err_create_qp_sq_size;
- }
-
- golan_qp->size = golan_qp->sq.size + golan_qp->rq.size;
-
- /* allocate dma memory for WQEs (1 page is enough) - should change it */
- golan_qp->wqes = malloc_dma ( GOLAN_PAGE_SIZE, GOLAN_PAGE_SIZE );
- if (!golan_qp->wqes) {
- rc = -ENOMEM;
- goto err_create_qp_wqe_alloc;
- }
- golan_qp->rq.wqes = golan_qp->wqes;
- golan_qp->sq.wqes = golan_qp->wqes + golan_qp->rq.size;//(union golan_send_wqe *)&
- //(((struct golan_recv_wqe_ud *)(golan_qp->wqes))[qp->recv.num_wqes]);
-
- if ( golan_qp->rq.grh_size ) {
- golan_qp->rq.grh = ( golan_qp->wqes +
- golan_qp->sq.size +
- golan_qp->rq.size );
- }
-
- /* Invalidate all WQEs */
- data = &golan_qp->rq.wqes[0].data[0];
- for ( i = 0 ; i < ( golan_qp->rq.size / sizeof ( *data ) ); i++ ){
- data->lkey = cpu_to_be32 ( GOLAN_INVALID_LKEY );
- data++;
- }
-
- golan_qp->doorbell_record = malloc_dma(sizeof(struct golan_qp_db),
- sizeof(struct golan_qp_db));
- if (!golan_qp->doorbell_record) {
- rc = -ENOMEM;
- goto err_create_qp_db_alloc;
- }
- memset(golan_qp->doorbell_record, 0, sizeof(struct golan_qp_db));
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_CREATE_QP, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_create_qp_mbox_in) + GOLAN_PAS_SIZE,
- sizeof(struct golan_create_qp_mbox_out));
-
- in = (struct golan_create_qp_mbox_in_data *)GET_INBOX(golan, GEN_MBOX);
-
- /* Fill the physical address of the page */
- in->pas[0] = VIRT_2_BE64_BUS(golan_qp->wqes);
- in->ctx.qp_counter_set_usr_page = cpu_to_be32(golan->uar.index);
-
- in->ctx.flags_pd = cpu_to_be32(golan->pdn);
- in->ctx.flags = cpu_to_be32((golan_qp_type_to_st(qp->type)
- << GOLAN_QP_CTX_ST_BIT) |
- (GOLAN_QP_PM_MIGRATED <<
- GOLAN_QP_CTX_PM_STATE_BIT));
- // cgs set to 0, initialy.
- // atomic mode
- in->ctx.rq_size_stride = ((ilog2(qp->recv.num_wqes) <<
- GOLAN_QP_CTX_RQ_SIZE_BIT) |
- (sizeof(golan_qp->rq.wqes[0]) / GOLAN_RECV_WQE_SIZE));
- in->ctx.sq_crq_size = cpu_to_be16(ilog2(golan_qp->sq.size / GOLAN_SEND_WQE_BB_SIZE)
- << GOLAN_QP_CTX_SQ_SIZE_BIT);
- in->ctx.cqn_send = cpu_to_be32(qp->send.cq->cqn);
- in->ctx.cqn_recv = cpu_to_be32(qp->recv.cq->cqn);
- in->ctx.db_rec_addr = VIRT_2_BE64_BUS(golan_qp->doorbell_record);
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_create_qp_cmd );
- out = (struct golan_create_qp_mbox_out *)cmd->out;
-
- *qpn = (be32_to_cpu(out->qpn) & 0xffffff);
- /*
- * Hardware wants QPN written in big-endian order (after
- * shifting) for send doorbell. Precompute this value to save
- * a little bit when posting sends.
- */
- golan_qp->doorbell_qpn = cpu_to_be32(*qpn << 8);
- golan_qp->state = GOLAN_IB_QPS_RESET;
-
- ib_qp_set_drvdata(qp, golan_qp);
-
- return 0;
-
- err_create_qp_cmd:
- free_dma(golan_qp->doorbell_record, sizeof(struct golan_qp_db));
- err_create_qp_db_alloc:
- free_dma ( golan_qp->wqes, GOLAN_PAGE_SIZE );
- err_create_qp_wqe_alloc:
- err_create_qp_sq_size:
- err_create_qp_sq_wqe_size:
- err_create_qp_rq_size:
- free ( golan_qp );
- err_create_qp:
- return rc;
- }
-
- /**
- * Create queue pair
- *
- * @v ibdev Infiniband device
- * @v qp Queue pair
- * @ret rc Return status code
- */
- static int golan_create_qp(struct ib_device *ibdev,
- struct ib_queue_pair *qp)
- {
- int rc, qpn = -1;
-
- switch (qp->type) {
- case IB_QPT_UD:
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- rc = golan_create_qp_aux(ibdev, qp, &qpn);
- if (rc) {
- DBG ( "%s Failed to create QP (rc = 0x%x)\n", __FUNCTION__, rc);
- return rc;
- }
- qp->qpn = qpn;
-
- break;
- case IB_QPT_ETH:
- case IB_QPT_RC:
- default:
- DBG ( "%s unsupported QP type (0x%x)\n", __FUNCTION__, qp->type);
- return -EINVAL;
- }
-
- return 0;
- }
-
- static int golan_modify_qp_rst_to_init(struct ib_device *ibdev,
- struct ib_queue_pair *qp __unused,
- struct golan_modify_qp_mbox_in_data *in)
- {
- int rc = 0;
-
- in->ctx.qkey = cpu_to_be32((uint32_t)(qp->qkey));
-
- in->ctx.pri_path.port = ibdev->port;
- in->ctx.flags |= cpu_to_be32(GOLAN_QP_PM_MIGRATED << GOLAN_QP_CTX_PM_STATE_BIT);
- in->ctx.pri_path.pkey_index = 0;
- /* QK is 0 */
- /* QP cntr set 0 */
- return rc;
- }
-
- static int golan_modify_qp_init_to_rtr(struct ib_device *ibdev __unused,
- struct ib_queue_pair *qp __unused,
- struct golan_modify_qp_mbox_in_data *in)
- {
- int rc = 0;
-
- in->optparam = 0;
- return rc;
- }
-
- static int golan_modify_qp_rtr_to_rts(struct ib_device *ibdev __unused,
- struct ib_queue_pair *qp __unused,
- struct golan_modify_qp_mbox_in_data *in __unused)
- {
- int rc = 0;
-
- in->optparam = 0;
- /* In good flow psn in 0 */
- return rc;
- }
-
- static int golan_modify_qp_to_rst(struct ib_device *ibdev,
- struct ib_queue_pair *qp)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_queue_pair *golan_qp = ib_qp_get_drvdata(qp);
- struct golan_cmd_layout *cmd;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_2RST_QP, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_modify_qp_mbox_in),
- sizeof(struct golan_modify_qp_mbox_out));
- ((struct golan_modify_qp_mbox_in *)(cmd->in))->qpn = cpu_to_be32(qp->qpn);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_modify_qp_2rst_cmd );
-
- golan_qp->state = GOLAN_IB_QPS_RESET;
- DBGC( golan , "%s QP number 0x%lx was modified to RESET\n",
- __FUNCTION__, qp->qpn);
-
- return 0;
-
- err_modify_qp_2rst_cmd:
- DBGC (golan ,"%s Failed to modify QP number 0x%lx (rc = 0x%x)\n",
- __FUNCTION__, qp->qpn, rc);
- return rc;
- }
-
- static int (*golan_modify_qp_methods[])(struct ib_device *ibdev,
- struct ib_queue_pair *qp,
- struct golan_modify_qp_mbox_in_data *in) = {
-
- [GOLAN_IB_QPS_RESET] = golan_modify_qp_rst_to_init,
- [GOLAN_IB_QPS_INIT] = golan_modify_qp_init_to_rtr,
- [GOLAN_IB_QPS_RTR] = golan_modify_qp_rtr_to_rts
- };
-
- static int golan_modify_qp(struct ib_device *ibdev,
- struct ib_queue_pair *qp)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_queue_pair *golan_qp = ib_qp_get_drvdata(qp);
- struct golan_modify_qp_mbox_in_data *in;
- struct golan_cmd_layout *cmd;
- enum golan_ib_qp_state prev_state;
- int rc;
- int modify_cmd[] = {GOLAN_CMD_OP_RST2INIT_QP,
- GOLAN_CMD_OP_INIT2RTR_QP,
- GOLAN_CMD_OP_RTR2RTS_QP};
-
- while (golan_qp->state < GOLAN_IB_QPS_RTS) {
- prev_state = golan_qp->state;
- cmd = write_cmd(golan, DEF_CMD_IDX, modify_cmd[golan_qp->state], 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_modify_qp_mbox_in),
- sizeof(struct golan_modify_qp_mbox_out));
-
- in = (struct golan_modify_qp_mbox_in_data *)GET_INBOX(golan, GEN_MBOX);
- ((struct golan_modify_qp_mbox_in *)(cmd->in))->qpn = cpu_to_be32(qp->qpn);
- rc = golan_modify_qp_methods[golan_qp->state](ibdev, qp, in);
- if (rc) {
- goto err_modify_qp_fill_inbox;
- }
- // in->ctx.qp_counter_set_usr_page = cpu_to_be32(golan->uar.index);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_modify_qp_cmd );
-
- ++(golan_qp->state);
-
- DBGC( golan , "%s QP number 0x%lx was modified from %s to %s\n",
- __FUNCTION__, qp->qpn, golan_qp_state_as_string[prev_state],
- golan_qp_state_as_string[golan_qp->state]);
- }
-
- DBGC( golan , "%s QP number 0x%lx is ready to receive/send packets.\n",
- __FUNCTION__, qp->qpn);
- return 0;
-
- err_modify_qp_cmd:
- err_modify_qp_fill_inbox:
- DBGC (golan ,"%s Failed to modify QP number 0x%lx (rc = 0x%x)\n",
- __FUNCTION__, qp->qpn, rc);
- return rc;
- }
-
- /**
- * Destroy queue pair
- *
- * @v ibdev Infiniband device
- * @v qp Queue pair
- */
- static void golan_destroy_qp(struct ib_device *ibdev,
- struct ib_queue_pair *qp)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_queue_pair *golan_qp = ib_qp_get_drvdata(qp);
- struct golan_cmd_layout *cmd;
- unsigned long qpn = qp->qpn;
- int rc;
-
- DBGC (golan, "%s in\n", __FUNCTION__);
-
- if (golan_qp->state != GOLAN_IB_QPS_RESET) {
- if (golan_modify_qp_to_rst(ibdev, qp)) {
- DBGC (golan ,"%s Failed to modify QP 0x%lx to RESET\n", __FUNCTION__,
- qp->qpn);
- }
- }
-
- if (qp->recv.cq) {
- golan_cq_clean(qp->recv.cq);
- }
- if (qp->send.cq && (qp->send.cq != qp->recv.cq)) {
- golan_cq_clean(qp->send.cq);
- }
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DESTROY_QP, 0x0,
- NO_MBOX, NO_MBOX,
- sizeof(struct golan_destroy_qp_mbox_in),
- sizeof(struct golan_destroy_qp_mbox_out));
- ((struct golan_destroy_qp_mbox_in *)(cmd->in))->qpn = cpu_to_be32(qpn);
- rc = send_command_and_wait(golan, DEF_CMD_IDX, NO_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
- qp->qpn = 0;
-
- ib_qp_set_drvdata(qp, NULL);
- free_dma(golan_qp->doorbell_record, sizeof(struct golan_qp_db));
- free_dma ( golan_qp->wqes, GOLAN_PAGE_SIZE );
- free(golan_qp);
-
- DBGC( golan ,"%s QP 0x%lx was destroyed\n", __FUNCTION__, qpn);
- }
-
- /**
- * Calculate transmission rate
- *
- * @v av Address vector
- * @ret golan_rate Golan rate
- */
- static unsigned int golan_rate(enum ib_rate rate) {
- return (((rate >= IB_RATE_2_5) && (rate <= IB_RATE_120)) ? (rate + 5) : 0);
- }
-
- /**
- * Post send work queue entry
- *
- * @v ibdev Infiniband device
- * @v qp Queue pair
- * @v av Address vector
- * @v iobuf I/O buffer
- * @ret rc Return status code
- */
- static int golan_post_send(struct ib_device *ibdev,
- struct ib_queue_pair *qp,
- struct ib_address_vector *av,
- struct io_buffer *iobuf)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_queue_pair *golan_qp = ib_qp_get_drvdata(qp);
- struct golan_send_wqe_ud *wqe = NULL;
- struct golan_av *datagram = NULL;
- unsigned long wqe_idx_mask;
- unsigned long wqe_idx;
- struct golan_wqe_data_seg *data = NULL;
- struct golan_wqe_ctrl_seg *ctrl = NULL;
-
-
- wqe_idx_mask = (qp->send.num_wqes - 1);
- wqe_idx = (qp->send.next_idx & wqe_idx_mask);
- if (qp->send.iobufs[wqe_idx]) {
- DBGC (golan ,"%s Send queue of QPN 0x%lx is full\n", __FUNCTION__, qp->qpn);
- return -ENOMEM;
- }
-
- qp->send.iobufs[wqe_idx] = iobuf;
-
- // change to this
- //wqe_size_in_octa_words = golan_qp->sq.wqe_size_in_wqebb >> 4;
-
- wqe = &golan_qp->sq.wqes[wqe_idx].ud;
-
- //CHECK HW OWNERSHIP BIT ???
-
- memset(wqe, 0, sizeof(*wqe));
-
- ctrl = &wqe->ctrl;
- ctrl->opmod_idx_opcode = cpu_to_be32(GOLAN_SEND_OPCODE |
- ((u32)(golan_qp->sq.next_idx) <<
- GOLAN_WQE_CTRL_WQE_IDX_BIT));
- ctrl->qpn_ds = cpu_to_be32(GOLAN_SEND_UD_WQE_SIZE >> 4) |
- golan_qp->doorbell_qpn;
- ctrl->fm_ce_se = 0x8;//10 - 0 - 0
- data = &wqe->data;
- data->byte_count = cpu_to_be32(iob_len(iobuf));
- data->lkey = cpu_to_be32(golan->mkey);
- data->addr = VIRT_2_BE64_BUS(iobuf->data);
-
- datagram = &wqe->datagram;
- datagram->key.qkey.qkey = cpu_to_be32(av->qkey);
- datagram->dqp_dct = cpu_to_be32((1 << 31) | av->qpn);
- datagram->stat_rate_sl = ((golan_rate(av->rate) << 4) | av->sl);
- datagram->fl_mlid = (ibdev->lid & 0x007f); /* take only the 7 low bits of the LID */
- datagram->rlid = cpu_to_be16(av->lid);
- datagram->grh_gid_fl = cpu_to_be32(av->gid_present << 30);
- memcpy(datagram->rgid, av->gid.bytes, 16 /* sizeof(datagram->rgid) */);
-
- /*
- * Make sure that descriptors are written before
- * updating doorbell record and ringing the doorbell
- */
- ++(qp->send.next_idx);
- golan_qp->sq.next_idx = (golan_qp->sq.next_idx + GOLAN_WQEBBS_PER_SEND_UD_WQE);
- golan_qp->doorbell_record->send_db = cpu_to_be16(golan_qp->sq.next_idx);
- wmb();
- writeq(*((__be64 *)ctrl), golan->uar.virt
- + ( ( golan_qp->sq.next_idx & 0x1 ) ? DB_BUFFER0_EVEN_OFFSET
- : DB_BUFFER0_ODD_OFFSET ) );
- return 0;
- }
-
- /**
- * Post receive work queue entry
- *
- * @v ibdev Infiniband device
- * @v qp Queue pair
- * @v iobuf I/O buffer
- * @ret rc Return status code
- */
- static int golan_post_recv(struct ib_device *ibdev,
- struct ib_queue_pair *qp,
- struct io_buffer *iobuf)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_queue_pair *golan_qp = ib_qp_get_drvdata(qp);
- struct ib_work_queue *wq = &qp->recv;
- struct golan_recv_wqe_ud *wqe;
- struct ib_global_route_header *grh;
- struct golan_wqe_data_seg *data;
- unsigned int wqe_idx_mask;
-
- /* Allocate work queue entry */
- wqe_idx_mask = (wq->num_wqes - 1);
- if (wq->iobufs[wq->next_idx & wqe_idx_mask]) {
- DBGC (golan ,"%s Receive queue of QPN 0x%lx is full\n", __FUNCTION__, qp->qpn);
- return -ENOMEM;
- }
-
- wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
- wqe = & golan_qp->rq.wqes[wq->next_idx & wqe_idx_mask];
-
- memset(wqe, 0, sizeof(*wqe));
- data = &wqe->data[0];
- if ( golan_qp->rq.grh ) {
- grh = &golan_qp->rq.grh[wq->next_idx & wqe_idx_mask];
- data->byte_count = cpu_to_be32 ( sizeof ( *grh ) );
- data->lkey = cpu_to_be32 ( golan->mkey );
- data->addr = VIRT_2_BE64_BUS ( grh );
- data++;
- }
-
- data->byte_count = cpu_to_be32(iob_tailroom(iobuf));
- data->lkey = cpu_to_be32(golan->mkey);
- data->addr = VIRT_2_BE64_BUS(iobuf->data);
-
- ++wq->next_idx;
-
- /*
- * Make sure that descriptors are written before
- * updating doorbell record and ringing the doorbell
- */
- wmb();
- golan_qp->doorbell_record->recv_db = cpu_to_be16(qp->recv.next_idx & 0xffff);
-
- return 0;
- }
-
- static int golan_query_vport_context ( struct ib_device *ibdev ) {
- struct golan *golan = ib_get_drvdata ( ibdev );
- struct golan_cmd_layout *cmd;
- struct golan_query_hca_vport_context_inbox *in;
- struct golan_query_hca_vport_context_data *context_data;
- int rc;
-
- cmd = write_cmd ( golan, DEF_CMD_IDX, GOLAN_CMD_OP_QUERY_HCA_VPORT_CONTEXT,
- 0x0, GEN_MBOX, GEN_MBOX,
- sizeof(struct golan_query_hca_vport_context_inbox),
- sizeof(struct golan_query_hca_vport_context_outbox) );
-
- in = GOLAN_MBOX_IN ( cmd, in );
- in->port_num = (u8)ibdev->port;
-
- rc = send_command_and_wait ( golan, DEF_CMD_IDX, GEN_MBOX, GEN_MBOX, __FUNCTION__ );
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_query_vport_context_cmd );
-
- context_data = (struct golan_query_hca_vport_context_data *)( GET_OUTBOX ( golan, GEN_MBOX ) );
-
- ibdev->node_guid.dwords[0] = context_data->node_guid[0];
- ibdev->node_guid.dwords[1] = context_data->node_guid[1];
- ibdev->lid = be16_to_cpu( context_data->lid );
- ibdev->sm_lid = be16_to_cpu( context_data->sm_lid );
- ibdev->sm_sl = context_data->sm_sl;
- ibdev->port_state = context_data->port_state;
-
- return 0;
- err_query_vport_context_cmd:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
-
- static int golan_query_vport_gid ( struct ib_device *ibdev ) {
- struct golan *golan = ib_get_drvdata( ibdev );
- struct golan_cmd_layout *cmd;
- struct golan_query_hca_vport_gid_inbox *in;
- union ib_gid *ib_gid;
- int rc;
-
- cmd = write_cmd( golan, DEF_CMD_IDX, GOLAN_CMD_OP_QUERY_HCA_VPORT_GID,
- 0x0, GEN_MBOX, GEN_MBOX,
- sizeof(struct golan_query_hca_vport_gid_inbox),
- sizeof(struct golan_query_hca_vport_gid_outbox) );
-
- in = GOLAN_MBOX_IN ( cmd, in );
- in->port_num = (u8)ibdev->port;
- in->gid_index = 0;
- rc = send_command_and_wait ( golan, DEF_CMD_IDX, GEN_MBOX, GEN_MBOX, __FUNCTION__ );
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_query_vport_gid_cmd );
-
- ib_gid = (union ib_gid *)( GET_OUTBOX ( golan, GEN_MBOX ) );
-
- memcpy ( &ibdev->gid, ib_gid, sizeof(ibdev->gid) );
-
- return 0;
- err_query_vport_gid_cmd:
- DBGC ( golan, "%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static int golan_query_vport_pkey ( struct ib_device *ibdev ) {
- struct golan *golan = ib_get_drvdata ( ibdev );
- struct golan_cmd_layout *cmd;
- struct golan_query_hca_vport_pkey_inbox *in;
- int pkey_table_size_in_entries = (1 << (7 + golan->caps.pkey_table_size));
- int rc;
-
- cmd = write_cmd ( golan, DEF_CMD_IDX, GOLAN_CMD_OP_QUERY_HCA_VPORT_PKEY,
- 0x0, GEN_MBOX, GEN_MBOX,
- sizeof(struct golan_query_hca_vport_pkey_inbox),
- sizeof(struct golan_outbox_hdr) + 8 +
- sizeof(struct golan_query_hca_vport_pkey_data) * pkey_table_size_in_entries );
-
- in = GOLAN_MBOX_IN ( cmd, in );
- in->port_num = (u8)ibdev->port;
- in->pkey_index = 0xffff;
- rc = send_command_and_wait ( golan, DEF_CMD_IDX, GEN_MBOX, GEN_MBOX, __FUNCTION__ );
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_query_vport_pkey_cmd );
-
- return 0;
- err_query_vport_pkey_cmd:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static int golan_get_ib_info ( struct ib_device *ibdev ) {
- int rc;
-
- rc = golan_query_vport_context ( ibdev );
- if ( rc != 0 ) {
- DBG ( "golan_get_ib_info: golan_query_vport_context Failed (rc = %d)\n",rc );
- goto err_query_vport_context;
- }
-
- rc = golan_query_vport_gid ( ibdev );
- if ( rc != 0 ) {
- DBG ( "golan_get_ib_info: golan_query_vport_gid Failed (rc = %d)\n",rc );
- goto err_query_vport_gid;
- }
-
- rc = golan_query_vport_pkey ( ibdev );
- if ( rc != 0 ) {
- DBG ( "golan_get_ib_info: golan_query_vport_pkey Failed (rc = %d)\n",rc );
- goto err_query_vport_pkey;
- }
- return rc;
- err_query_vport_pkey:
- err_query_vport_gid:
- err_query_vport_context:
- DBG ( "%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- static int golan_complete(struct ib_device *ibdev,
- struct ib_completion_queue *cq,
- struct golan_cqe64 *cqe64)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct ib_work_queue *wq;
- struct golan_queue_pair *golan_qp;
- struct ib_queue_pair *qp;
- struct io_buffer *iobuf = NULL;
- struct ib_address_vector recv_dest;
- struct ib_address_vector recv_source;
- struct ib_global_route_header *grh;
- struct golan_err_cqe *err_cqe64;
- int gid_present, idx;
- u16 wqe_ctr;
- uint8_t opcode;
- static int error_state;
- uint32_t qpn = be32_to_cpu(cqe64->sop_drop_qpn) & 0xffffff;
- int is_send = 0;
- size_t len;
-
- opcode = cqe64->op_own >> GOLAN_CQE_OPCODE_BIT;
- DBGC2( golan , "%s completion with opcode 0x%x\n", __FUNCTION__, opcode);
-
- if (opcode == GOLAN_CQE_REQ || opcode == GOLAN_CQE_REQ_ERR) {
- is_send = 1;
- } else {
- is_send = 0;
- }
- if (opcode == GOLAN_CQE_REQ_ERR || opcode == GOLAN_CQE_RESP_ERR) {
- err_cqe64 = (struct golan_err_cqe *)cqe64;
- int i = 0;
- if (!error_state++) {
- DBGC (golan ,"\n");
- for ( i = 0 ; i < 16 ; i += 2 ) {
- DBGC (golan ,"%x %x\n",
- be32_to_cpu(((uint32_t *)(err_cqe64))[i]),
- be32_to_cpu(((uint32_t *)(err_cqe64))[i + 1]));
- }
- DBGC (golan ,"CQE with error: Syndrome(0x%x), VendorSynd(0x%x), HW_SYN(0x%x)\n",
- err_cqe64->syndrome, err_cqe64->vendor_err_synd,
- err_cqe64->hw_syndrom);
- }
- }
- /* Identify work queue */
- wq = ib_find_wq(cq, qpn, is_send);
- if (!wq) {
- DBGC (golan ,"%s unknown %s QPN 0x%x in CQN 0x%lx\n",
- __FUNCTION__, (is_send ? "send" : "recv"), qpn, cq->cqn);
- return -EINVAL;
- }
-
- qp = wq->qp;
- golan_qp = ib_qp_get_drvdata ( qp );
-
- wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
- if (is_send) {
- wqe_ctr &= ((GOLAN_WQEBBS_PER_SEND_UD_WQE * wq->num_wqes) - 1);
- idx = wqe_ctr / GOLAN_WQEBBS_PER_SEND_UD_WQE;
- } else {
- idx = wqe_ctr & (wq->num_wqes - 1);
- }
-
- iobuf = wq->iobufs[idx];
- if (!iobuf) {
- DBGC (golan ,"%s IO Buffer 0x%x not found in QPN 0x%x\n",
- __FUNCTION__, idx, qpn);
- return -EINVAL;
- }
- wq->iobufs[idx] = NULL;
-
- if (is_send) {
- ib_complete_send(ibdev, qp, iobuf, (opcode == GOLAN_CQE_REQ_ERR));
- } else {
- len = be32_to_cpu(cqe64->byte_cnt);
- memset(&recv_dest, 0, sizeof(recv_dest));
- recv_dest.qpn = qpn;
- /* Construct address vector */
- memset(&recv_source, 0, sizeof(recv_source));
- switch (qp->type) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- case IB_QPT_UD:
- /* Locate corresponding GRH */
- assert ( golan_qp->rq.grh != NULL );
- grh = &golan_qp->rq.grh[ idx ];
-
- recv_source.qpn = be32_to_cpu(cqe64->flags_rqpn) & 0xffffff;
- recv_source.lid = be16_to_cpu(cqe64->slid);
- recv_source.sl = (be32_to_cpu(cqe64->flags_rqpn) >> 24) & 0xf;
- gid_present = (be32_to_cpu(cqe64->flags_rqpn) >> 28) & 3;
- if (!gid_present) {
- recv_dest.gid_present = recv_source.gid_present = 0;
- } else {
- recv_dest.gid_present = recv_source.gid_present = 1;
- //if (recv_source.gid_present == 0x1) {
- memcpy(&recv_source.gid, &grh->sgid, sizeof(recv_source.gid));
- memcpy(&recv_dest.gid, &grh->dgid, sizeof(recv_dest.gid));
- //} else { // recv_source.gid_present = 0x3
- /* GRH is located in the upper 64 byte of the CQE128
- * currently not supported */
- //;
- //}
- }
- len -= sizeof ( *grh );
- break;
- case IB_QPT_RC:
- case IB_QPT_ETH:
- default:
- DBGC (golan ,"%s Unsupported QP type (0x%x)\n", __FUNCTION__, qp->type);
- return -EINVAL;
- }
- assert(len <= iob_tailroom(iobuf));
- iob_put(iobuf, len);
- ib_complete_recv(ibdev, qp, &recv_dest, &recv_source, iobuf, (opcode == GOLAN_CQE_RESP_ERR));
- }
- return 0;
- }
-
- static int golan_is_hw_ownership(struct ib_completion_queue *cq,
- struct golan_cqe64 *cqe64)
- {
- return ((cqe64->op_own & GOLAN_CQE_OWNER_MASK) !=
- ((cq->next_idx >> ilog2(cq->num_cqes)) & 1));
- }
- static void golan_poll_cq(struct ib_device *ibdev,
- struct ib_completion_queue *cq)
- {
- unsigned int i;
- int rc = 0;
- unsigned int cqe_idx_mask;
- struct golan_cqe64 *cqe64;
- struct golan_completion_queue *golan_cq = ib_cq_get_drvdata(cq);
- struct golan *golan = ib_get_drvdata(ibdev);
-
- for (i = 0; i < cq->num_cqes; ++i) {
- /* Look for completion entry */
- cqe_idx_mask = (cq->num_cqes - 1);
- cqe64 = &golan_cq->cqes[cq->next_idx & cqe_idx_mask];
- /* temporary valid only for 64 byte CQE */
- if (golan_is_hw_ownership(cq, cqe64) ||
- ((cqe64->op_own >> GOLAN_CQE_OPCODE_BIT) ==
- GOLAN_CQE_OPCODE_NOT_VALID)) {
- break; /* HW ownership */
- }
-
- DBGC2( golan , "%s CQN 0x%lx [%ld] \n", __FUNCTION__, cq->cqn, cq->next_idx);
- /*
- * Make sure we read CQ entry contents after we've checked the
- * ownership bit. (PRM - 6.5.3.2)
- */
- rmb();
- rc = golan_complete(ibdev, cq, cqe64);
- if (rc != 0) {
- DBGC (golan ,"%s CQN 0x%lx failed to complete\n", __FUNCTION__, cq->cqn);
- }
-
- /* Update completion queue's index */
- cq->next_idx++;
-
- /* Update doorbell record */
- *(golan_cq->doorbell_record) = cpu_to_be32(cq->next_idx & 0xffffff);
- }
- }
-
- static const char *golan_eqe_type_str(u8 type)
- {
- switch (type) {
- case GOLAN_EVENT_TYPE_COMP:
- return "GOLAN_EVENT_TYPE_COMP";
- case GOLAN_EVENT_TYPE_PATH_MIG:
- return "GOLAN_EVENT_TYPE_PATH_MIG";
- case GOLAN_EVENT_TYPE_COMM_EST:
- return "GOLAN_EVENT_TYPE_COMM_EST";
- case GOLAN_EVENT_TYPE_SQ_DRAINED:
- return "GOLAN_EVENT_TYPE_SQ_DRAINED";
- case GOLAN_EVENT_TYPE_SRQ_LAST_WQE:
- return "GOLAN_EVENT_TYPE_SRQ_LAST_WQE";
- case GOLAN_EVENT_TYPE_SRQ_RQ_LIMIT:
- return "GOLAN_EVENT_TYPE_SRQ_RQ_LIMIT";
- case GOLAN_EVENT_TYPE_CQ_ERROR:
- return "GOLAN_EVENT_TYPE_CQ_ERROR";
- case GOLAN_EVENT_TYPE_WQ_CATAS_ERROR:
- return "GOLAN_EVENT_TYPE_WQ_CATAS_ERROR";
- case GOLAN_EVENT_TYPE_PATH_MIG_FAILED:
- return "GOLAN_EVENT_TYPE_PATH_MIG_FAILED";
- case GOLAN_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
- return "GOLAN_EVENT_TYPE_WQ_INVAL_REQ_ERROR";
- case GOLAN_EVENT_TYPE_WQ_ACCESS_ERROR:
- return "GOLAN_EVENT_TYPE_WQ_ACCESS_ERROR";
- case GOLAN_EVENT_TYPE_SRQ_CATAS_ERROR:
- return "GOLAN_EVENT_TYPE_SRQ_CATAS_ERROR";
- case GOLAN_EVENT_TYPE_INTERNAL_ERROR:
- return "GOLAN_EVENT_TYPE_INTERNAL_ERROR";
- case GOLAN_EVENT_TYPE_PORT_CHANGE:
- return "GOLAN_EVENT_TYPE_PORT_CHANGE";
- case GOLAN_EVENT_TYPE_GPIO_EVENT:
- return "GOLAN_EVENT_TYPE_GPIO_EVENT";
- case GOLAN_EVENT_TYPE_REMOTE_CONFIG:
- return "GOLAN_EVENT_TYPE_REMOTE_CONFIG";
- case GOLAN_EVENT_TYPE_DB_BF_CONGESTION:
- return "GOLAN_EVENT_TYPE_DB_BF_CONGESTION";
- case GOLAN_EVENT_TYPE_STALL_EVENT:
- return "GOLAN_EVENT_TYPE_STALL_EVENT";
- case GOLAN_EVENT_TYPE_CMD:
- return "GOLAN_EVENT_TYPE_CMD";
- case GOLAN_EVENT_TYPE_PAGE_REQUEST:
- return "GOLAN_EVENT_TYPE_PAGE_REQUEST";
- default:
- return "Unrecognized event";
- }
- }
-
- static const char *golan_eqe_port_subtype_str(u8 subtype)
- {
- switch (subtype) {
- case GOLAN_PORT_CHANGE_SUBTYPE_DOWN:
- return "GOLAN_PORT_CHANGE_SUBTYPE_DOWN";
- case GOLAN_PORT_CHANGE_SUBTYPE_ACTIVE:
- return "GOLAN_PORT_CHANGE_SUBTYPE_ACTIVE";
- case GOLAN_PORT_CHANGE_SUBTYPE_INITIALIZED:
- return "GOLAN_PORT_CHANGE_SUBTYPE_INITIALIZED";
- case GOLAN_PORT_CHANGE_SUBTYPE_LID:
- return "GOLAN_PORT_CHANGE_SUBTYPE_LID";
- case GOLAN_PORT_CHANGE_SUBTYPE_PKEY:
- return "GOLAN_PORT_CHANGE_SUBTYPE_PKEY";
- case GOLAN_PORT_CHANGE_SUBTYPE_GUID:
- return "GOLAN_PORT_CHANGE_SUBTYPE_GUID";
- case GOLAN_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
- return "GOLAN_PORT_CHANGE_SUBTYPE_CLIENT_REREG";
- default:
- return "Unrecognized event";
- }
- }
-
- /**
- * Update Infiniband parameters using Commands
- *
- * @v ibdev Infiniband device
- * @ret rc Return status code
- */
- static int golan_ib_update ( struct ib_device *ibdev ) {
- int rc;
-
- /* Get IB parameters */
- if ( ( rc = golan_get_ib_info ( ibdev ) ) != 0 )
- return rc;
-
- /* Notify Infiniband core of potential link state change */
- ib_link_state_changed ( ibdev );
-
- return 0;
- }
-
- static inline void golan_handle_port_event(struct golan *golan, struct golan_eqe *eqe)
- {
- struct ib_device *ibdev;
- u8 port;
-
- port = (eqe->data.port.port >> 4) & 0xf;
- ibdev = golan->ports[port - 1].ibdev;
-
- if ( ! ib_is_open ( ibdev ) )
- return;
-
- switch (eqe->sub_type) {
- case GOLAN_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
- case GOLAN_PORT_CHANGE_SUBTYPE_ACTIVE:
- golan_ib_update ( ibdev );
- /* Fall through */
- case GOLAN_PORT_CHANGE_SUBTYPE_DOWN:
- case GOLAN_PORT_CHANGE_SUBTYPE_LID:
- case GOLAN_PORT_CHANGE_SUBTYPE_PKEY:
- case GOLAN_PORT_CHANGE_SUBTYPE_GUID:
- case GOLAN_PORT_CHANGE_SUBTYPE_INITIALIZED:
- DBGC( golan , "%s event %s(%d) (sub event %s(%d))arrived on port %d\n",
- __FUNCTION__, golan_eqe_type_str(eqe->type), eqe->type,
- golan_eqe_port_subtype_str(eqe->sub_type),
- eqe->sub_type, port);
- break;
- default:
- DBGC (golan ,"%s Port event with unrecognized subtype: port %d, sub_type %d\n",
- __FUNCTION__, port, eqe->sub_type);
- }
- }
-
- static struct golan_eqe *golan_next_eqe_sw(struct golan_event_queue *eq)
- {
- uint32_t entry = (eq->cons_index & (GOLAN_NUM_EQES - 1));
- struct golan_eqe *eqe = &(eq->eqes[entry]);
- return ((eqe->owner != ((eq->cons_index >> ilog2(GOLAN_NUM_EQES)) & 1)) ? NULL : eqe);
- }
-
-
- /**
- * Poll event queue
- *
- * @v ibdev Infiniband device
- */
- static void golan_poll_eq(struct ib_device *ibdev)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_event_queue *eq = &(golan->eq);
- struct golan_eqe *eqe;
- u32 cqn;
- int counter = 0;
-
- while ((eqe = golan_next_eqe_sw(eq)) && (counter < GOLAN_NUM_EQES)) {
- /*
- * Make sure we read EQ entry contents after we've
- * checked the ownership bit.
- */
- rmb();
-
- DBGC( golan , "%s eqn %d, eqe type %s\n", __FUNCTION__, eq->eqn,
- golan_eqe_type_str(eqe->type));
- switch (eqe->type) {
- case GOLAN_EVENT_TYPE_COMP:
- /* We dont need to handle completion events since we
- * poll all the CQs after polling the EQ */
- break;
- case GOLAN_EVENT_TYPE_PATH_MIG:
- case GOLAN_EVENT_TYPE_COMM_EST:
- case GOLAN_EVENT_TYPE_SQ_DRAINED:
- case GOLAN_EVENT_TYPE_SRQ_LAST_WQE:
- case GOLAN_EVENT_TYPE_WQ_CATAS_ERROR:
- case GOLAN_EVENT_TYPE_PATH_MIG_FAILED:
- case GOLAN_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
- case GOLAN_EVENT_TYPE_WQ_ACCESS_ERROR:
- case GOLAN_EVENT_TYPE_SRQ_RQ_LIMIT:
- case GOLAN_EVENT_TYPE_SRQ_CATAS_ERROR:
- DBGC( golan , "%s event %s(%d) arrived\n", __FUNCTION__,
- golan_eqe_type_str(eqe->type), eqe->type);
- break;
- case GOLAN_EVENT_TYPE_CMD:
- // golan_cmd_comp_handler(be32_to_cpu(eqe->data.cmd.vector));
- break;
- case GOLAN_EVENT_TYPE_PORT_CHANGE:
- golan_handle_port_event(golan, eqe);
- break;
- case GOLAN_EVENT_TYPE_CQ_ERROR:
- cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
- DBGC (golan ,"CQ error on CQN 0x%x, syndrom 0x%x\n",
- cqn, eqe->data.cq_err.syndrome);
- // mlx5_cq_event(dev, cqn, eqe->type);
- break;
- /*
- * currently the driver do not support dynamic memory request
- * during FW run, a follow up change will allocate FW pages once and
- * never release them till driver shutdown, this change will not support
- * this request as currently this request is not issued anyway.
- case GOLAN_EVENT_TYPE_PAGE_REQUEST:
- {
- // we should check if we get this event while we
- // waiting for a command
- u16 func_id = be16_to_cpu(eqe->data.req_pages.func_id);
- s16 npages = be16_to_cpu(eqe->data.req_pages.num_pages);
-
- DBGC (golan ,"%s page request for func 0x%x, napges %d\n",
- __FUNCTION__, func_id, npages);
- golan_provide_pages(golan, npages, func_id);
- }
- break;
- */
- default:
- DBGC (golan ,"%s Unhandled event 0x%x on EQ 0x%x\n", __FUNCTION__,
- eqe->type, eq->eqn);
- break;
- }
-
- ++eq->cons_index;
- golan_eq_update_ci(eq, GOLAN_EQ_UNARMED);
- ++counter;
- }
- }
-
- /**
- * Attach to multicast group
- *
- * @v ibdev Infiniband device
- * @v qp Queue pair
- * @v gid Multicast GID
- * @ret rc Return status code
- */
- static int golan_mcast_attach(struct ib_device *ibdev,
- struct ib_queue_pair *qp,
- union ib_gid *gid)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_cmd_layout *cmd;
- int rc;
-
- if ( qp == NULL ) {
- DBGC( golan, "%s: Invalid pointer, could not attach QPN to MCG\n",
- __FUNCTION__ );
- return -EFAULT;
- }
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_ATTACH_TO_MCG, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_attach_mcg_mbox_in),
- sizeof(struct golan_attach_mcg_mbox_out));
- ((struct golan_attach_mcg_mbox_in *)(cmd->in))->qpn = cpu_to_be32(qp->qpn);
-
- memcpy(GET_INBOX(golan, GEN_MBOX), gid, sizeof(*gid));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_CHECK_RC_AND_CMD_STATUS( err_attach_to_mcg_cmd );
-
- DBGC( golan , "%s: QPN 0x%lx was attached to MCG\n", __FUNCTION__, qp->qpn);
- return 0;
- err_attach_to_mcg_cmd:
- DBGC (golan ,"%s [%d] out\n", __FUNCTION__, rc);
- return rc;
- }
-
- /**
- * Detach from multicast group
- *
- * @v ibdev Infiniband device
- * @v qp Queue pair
- * @v gid Multicast GID
- * @ret rc Return status code
- */
- static void golan_mcast_detach(struct ib_device *ibdev,
- struct ib_queue_pair *qp,
- union ib_gid *gid)
- {
- struct golan *golan = ib_get_drvdata(ibdev);
- struct golan_cmd_layout *cmd;
- int rc;
-
- cmd = write_cmd(golan, DEF_CMD_IDX, GOLAN_CMD_OP_DETACH_FROM_MCG, 0x0,
- GEN_MBOX, NO_MBOX,
- sizeof(struct golan_detach_mcg_mbox_in),
- sizeof(struct golan_detach_mcg_mbox_out));
- ((struct golan_detach_mcg_mbox_in *)(cmd->in))->qpn = cpu_to_be32(qp->qpn);
-
- memcpy(GET_INBOX(golan, GEN_MBOX), gid, sizeof(*gid));
-
- rc = send_command_and_wait(golan, DEF_CMD_IDX, GEN_MBOX, NO_MBOX, __FUNCTION__);
- GOLAN_PRINT_RC_AND_CMD_STATUS;
-
- DBGC( golan , "%s: QPN 0x%lx was detached from MCG\n", __FUNCTION__, qp->qpn);
- }
-
- /**
- * Inform embedded subnet management agent of a received MAD
- *
- * @v ibdev Infiniband device
- * @v mad MAD
- * @ret rc Return status code
- */
- static int golan_inform_sma(struct ib_device *ibdev,
- union ib_mad *mad)
- {
- if (!ibdev || !mad) {
- return 1;
- }
-
- return 0;
- }
-
- static int golan_register_ibdev(struct golan_port *port)
- {
- struct ib_device *ibdev = port->ibdev;
- int rc;
-
- golan_get_ib_info ( ibdev );
- /* Register Infiniband device */
- if ((rc = register_ibdev(ibdev)) != 0) {
- DBG ( "%s port %d could not register IB device: (rc = %d)\n",
- __FUNCTION__, ibdev->port, rc);
- return rc;
- }
-
- port->netdev = ipoib_netdev( ibdev );
-
- return 0;
- }
-
- static inline void golan_bring_down(struct golan *golan)
- {
- DBGC(golan, "%s: start\n", __FUNCTION__);
-
- if (~golan->flags & GOLAN_OPEN) {
- DBGC(golan, "%s: end (already closed)\n", __FUNCTION__);
- return;
- }
-
- golan_destroy_mkey(golan);
- golan_dealloc_pd(golan);
- golan_destory_eq(golan);
- golan_dealloc_uar(golan);
- golan_teardown_hca(golan, GOLAN_TEARDOWN_GRACEFUL);
- golan_handle_pages(golan, GOLAN_REG_PAGES , GOLAN_PAGES_TAKE);
- golan_disable_hca(golan);
- golan_cmd_uninit(golan);
- golan->flags &= ~GOLAN_OPEN;
- DBGC(golan, "%s: end\n", __FUNCTION__);
- }
-
- static int golan_set_link_speed ( struct golan *golan ){
- mlx_status status;
- int i = 0;
- int utils_inited = 0;
-
- if ( ! golan->utils ) {
- utils_inited = 1;
- status = init_mlx_utils ( & golan->utils, golan->pci );
- MLX_CHECK_STATUS ( golan->pci, status, utils_init_err, "mlx_utils_init failed" );
- }
-
- for ( i = 0; i < golan->caps.num_ports; ++i ) {
- status = mlx_set_link_speed ( golan->utils, i + 1, LINK_SPEED_IB, LINK_SPEED_SDR );
- MLX_CHECK_STATUS ( golan->pci, status, set_link_speed_err, "mlx_set_link_speed failed" );
- }
-
- set_link_speed_err:
- if ( utils_inited )
- free_mlx_utils ( & golan->utils );
- utils_init_err:
- return status;
- }
-
- static inline int golan_bring_up(struct golan *golan)
- {
- int rc = 0;
- DBGC(golan, "%s\n", __FUNCTION__);
-
- if (golan->flags & GOLAN_OPEN)
- return 0;
-
- if (( rc = golan_cmd_init(golan) ))
- goto out;
-
- if (( rc = golan_core_enable_hca(golan) ))
- goto cmd_uninit;
-
- /* Query for need for boot pages */
- if (( rc = golan_handle_pages(golan, GOLAN_BOOT_PAGES, GOLAN_PAGES_GIVE) ))
- goto disable;
-
- if (( rc = golan_qry_hca_cap(golan) ))
- goto pages;
-
- if (( rc = golan_set_hca_cap(golan) ))
- goto pages;
-
- if (( rc = golan_handle_pages(golan, GOLAN_INIT_PAGES, GOLAN_PAGES_GIVE) ))
- goto pages;
-
- if (( rc = golan_set_link_speed ( golan ) ))
- goto pages_teardown;
-
- //Reg Init?
- if (( rc = golan_hca_init(golan) ))
- goto pages_2;
-
- if (( rc = golan_alloc_uar(golan) ))
- goto teardown;
-
- if (( rc = golan_create_eq(golan) ))
- goto de_uar;
-
- if (( rc = golan_alloc_pd(golan) ))
- goto de_eq;
-
- if (( rc = golan_create_mkey(golan) ))
- goto de_pd;
-
- golan->flags |= GOLAN_OPEN;
- return 0;
-
- golan_destroy_mkey(golan);
- de_pd:
- golan_dealloc_pd(golan);
- de_eq:
- golan_destory_eq(golan);
- de_uar:
- golan_dealloc_uar(golan);
- teardown:
- golan_teardown_hca(golan, GOLAN_TEARDOWN_GRACEFUL);
- pages_2:
- pages_teardown:
- golan_handle_pages(golan, GOLAN_INIT_PAGES, GOLAN_PAGES_TAKE);
- pages:
- golan_handle_pages(golan, GOLAN_BOOT_PAGES, GOLAN_PAGES_TAKE);
- disable:
- golan_disable_hca(golan);
- cmd_uninit:
- golan_cmd_uninit(golan);
- out:
- return rc;
- }
-
- /**
- * Close Infiniband link
- *
- * @v ibdev Infiniband device
- */
- static void golan_ib_close ( struct ib_device *ibdev ) {
- struct golan *golan = NULL;
-
- DBG ( "%s start\n", __FUNCTION__ );
- if ( ! ibdev )
- return;
- golan = ib_get_drvdata ( ibdev );
- golan_bring_down ( golan );
- DBG ( "%s end\n", __FUNCTION__ );
- }
-
- /**
- * Initialise Infiniband link
- *
- * @v ibdev Infiniband device
- * @ret rc Return status code
- */
- static int golan_ib_open ( struct ib_device *ibdev ) {
- struct golan *golan = NULL;
- DBG ( "%s start\n", __FUNCTION__ );
-
- if ( ! ibdev )
- return -EINVAL;
- golan = ib_get_drvdata ( ibdev );
- golan_bring_up ( golan );
- golan_ib_update ( ibdev );
-
- DBG ( "%s end\n", __FUNCTION__ );
- return 0;
- }
-
- /** Golan Infiniband operations */
- static struct ib_device_operations golan_ib_operations = {
- .create_cq = golan_create_cq,
- .destroy_cq = golan_destroy_cq,
- .create_qp = golan_create_qp,
- .modify_qp = golan_modify_qp,
- .destroy_qp = golan_destroy_qp,
- .post_send = golan_post_send,
- .post_recv = golan_post_recv,
- .poll_cq = golan_poll_cq,
- .poll_eq = golan_poll_eq,
- .open = golan_ib_open,
- .close = golan_ib_close,
- .mcast_attach = golan_mcast_attach,
- .mcast_detach = golan_mcast_detach,
- .set_port_info = golan_inform_sma,
- .set_pkey_table = golan_inform_sma,
- };
-
- static int golan_probe_normal ( struct pci_device *pci ) {
- struct golan *golan;
- struct ib_device *ibdev;
- struct golan_port *port;
- int i;
- int rc = 0;
-
- golan = golan_alloc();
- if ( !golan ) {
- rc = -ENOMEM;
- goto err_golan_alloc;
- }
-
- /* at POST stage some BIOSes have limited available dynamic memory */
- if ( golan_init_fw_areas ( golan ) ) {
- rc = -ENOMEM;
- goto err_golan_golan_init_pages;
- }
-
- /* Setup PCI bus and HCA BAR */
- pci_set_drvdata( pci, golan );
- golan->pci = pci;
- golan_pci_init( golan );
- /* config command queues */
- if ( golan_bring_up( golan ) ) {
- DBGC (golan ,"golan bringup failed\n");
- rc = -1;
- goto err_golan_bringup;
- }
-
- if ( ! DEVICE_IS_CIB ( pci->device ) ) {
- if ( init_mlx_utils ( & golan->utils, pci ) ) {
- rc = -1;
- goto err_utils_init;
- }
- }
- /* Allocate Infiniband devices */
- for (i = 0; i < golan->caps.num_ports; ++i) {
- ibdev = alloc_ibdev( 0 );
- if ( !ibdev ) {
- rc = -ENOMEM;
- goto err_golan_probe_alloc_ibdev;
- }
- golan->ports[i].ibdev = ibdev;
- golan->ports[i].vep_number = 0;
- ibdev->op = &golan_ib_operations;
- ibdev->dev = &pci->dev;
- ibdev->port = (GOLAN_PORT_BASE + i);
- ib_set_drvdata( ibdev, golan );
- }
-
- /* Register devices */
- for ( i = 0; i < golan->caps.num_ports; ++i ) {
- port = &golan->ports[i];
- if ((rc = golan_register_ibdev ( port ) ) != 0 ) {
- goto err_golan_probe_register_ibdev;
- }
- }
-
- golan_bring_down ( golan );
-
- return 0;
-
- i = golan->caps.num_ports;
- err_golan_probe_register_ibdev:
- for ( i-- ; ( signed int ) i >= 0 ; i-- )
- unregister_ibdev ( golan->ports[i].ibdev );
-
- i = golan->caps.num_ports;
- err_golan_probe_alloc_ibdev:
- for ( i-- ; ( signed int ) i >= 0 ; i-- )
- ibdev_put ( golan->ports[i].ibdev );
- if ( ! DEVICE_IS_CIB ( pci->device ) ) {
- free_mlx_utils ( & golan->utils );
- }
- err_utils_init:
- golan_bring_down ( golan );
- err_golan_bringup:
- iounmap( golan->iseg );
- golan_free_fw_areas ( golan );
- err_golan_golan_init_pages:
- free ( golan );
- err_golan_alloc:
- DBGC (golan ,"%s rc = %d\n", __FUNCTION__, rc);
- return rc;
- }
-
- static void golan_remove_normal ( struct pci_device *pci ) {
- struct golan *golan = pci_get_drvdata(pci);
- struct golan_port *port;
- int i;
-
- DBGC(golan, "%s\n", __FUNCTION__);
-
- for ( i = ( golan->caps.num_ports - 1 ) ; i >= 0 ; i-- ) {
- port = &golan->ports[i];
- unregister_ibdev ( port->ibdev );
- }
- for ( i = ( golan->caps.num_ports - 1 ) ; i >= 0 ; i-- ) {
- netdev_nullify ( golan->ports[i].netdev );
- }
- for ( i = ( golan->caps.num_ports - 1 ) ; i >= 0 ; i-- ) {
- ibdev_put ( golan->ports[i].ibdev );
- }
- if ( ! DEVICE_IS_CIB ( pci->device ) ) {
- free_mlx_utils ( & golan->utils );
- }
- iounmap( golan->iseg );
- golan_free_fw_areas ( golan );
- free(golan);
- }
-
- /***************************************************************************
- * NODNIC operations
- **************************************************************************/
- static mlx_status shomron_tx_uar_send_db ( struct ib_device *ibdev,
- struct nodnic_send_wqbb *wqbb ) {
- mlx_status status = MLX_SUCCESS;
- struct flexboot_nodnic *flexboot_nodnic = ib_get_drvdata ( ibdev );
- struct shomron_nodnic_eth_send_wqe *eth_wqe =
- ( struct shomron_nodnic_eth_send_wqe * )wqbb;
- struct shomronprm_wqe_segment_ctrl_send *ctrl;
-
- if ( ! eth_wqe || ! flexboot_nodnic->device_priv.uar.virt ) {
- DBG("%s: Invalid parameters\n",__FUNCTION__);
- status = MLX_FAILED;
- goto err;
- }
- wmb();
- ctrl = & eth_wqe->ctrl;
- writeq(*((__be64 *)ctrl), flexboot_nodnic->device_priv.uar.virt +
- ( ( MLX_GET ( ctrl, wqe_index ) & 0x1 ) ? DB_BUFFER0_ODD_OFFSET
- : DB_BUFFER0_EVEN_OFFSET ) );
- err:
- return status;
- }
-
- static mlx_status shomron_fill_eth_send_wqe ( struct ib_device *ibdev,
- struct ib_queue_pair *qp, struct ib_address_vector *av __unused,
- struct io_buffer *iobuf, struct nodnic_send_wqbb *wqbb,
- unsigned long wqe_index ) {
- mlx_status status = MLX_SUCCESS;
- struct flexboot_nodnic *flexboot_nodnic = ib_get_drvdata ( ibdev );
- struct shomron_nodnic_eth_send_wqe *eth_wqe = NULL;
- struct flexboot_nodnic_port *port = &flexboot_nodnic->port[ibdev->port - 1];
- struct flexboot_nodnic_queue_pair *flexboot_nodnic_qp =
- ib_qp_get_drvdata ( qp );
- nodnic_qp *nodnic_qp = flexboot_nodnic_qp->nodnic_queue_pair;
- struct nodnic_send_ring *send_ring = &nodnic_qp->send;
- mlx_uint32 qpn = 0;
-
- eth_wqe = (struct shomron_nodnic_eth_send_wqe *)wqbb;
- memset ( ( ( ( void * ) eth_wqe ) ), 0,
- ( sizeof ( *eth_wqe ) ) );
-
- status = nodnic_port_get_qpn(&port->port_priv, &send_ring->nodnic_ring,
- &qpn);
- if ( status != MLX_SUCCESS ) {
- DBG("nodnic_port_get_qpn failed\n");
- goto err;
- }
-
- #define SHOMRON_GENERATE_CQE 0x3
- #define SHOMRON_INLINE_HEADERS_SIZE 18
- #define SHOMRON_INLINE_HEADERS_OFFSET 32
- MLX_FILL_2 ( ð_wqe->ctrl, 0, opcode, FLEXBOOT_NODNIC_OPCODE_SEND,
- wqe_index, wqe_index & 0xFFFF);
- MLX_FILL_2 ( ð_wqe->ctrl, 1, ds, 0x4 , qpn, qpn );
- MLX_FILL_1 ( ð_wqe->ctrl, 2,
- ce, SHOMRON_GENERATE_CQE /* generate completion */
- );
- MLX_FILL_2 ( ð_wqe->ctrl, 7,
- inline_headers1,
- cpu_to_be16(*(mlx_uint16 *)iobuf->data),
- inline_headers_size, SHOMRON_INLINE_HEADERS_SIZE
- );
- memcpy((void *)ð_wqe->ctrl + SHOMRON_INLINE_HEADERS_OFFSET,
- iobuf->data + 2, SHOMRON_INLINE_HEADERS_SIZE - 2);
- iob_pull(iobuf, SHOMRON_INLINE_HEADERS_SIZE);
- MLX_FILL_1 ( ð_wqe->data[0], 0,
- byte_count, iob_len ( iobuf ) );
- MLX_FILL_1 ( ð_wqe->data[0], 1, l_key,
- flexboot_nodnic->device_priv.lkey );
- MLX_FILL_H ( ð_wqe->data[0], 2,
- local_address_h, virt_to_bus ( iobuf->data ) );
- MLX_FILL_1 ( ð_wqe->data[0], 3,
- local_address_l, virt_to_bus ( iobuf->data ) );
- err:
- return status;
- }
-
- static mlx_status shomron_fill_completion( void *cqe, struct cqe_data *cqe_data ) {
- union shomronprm_completion_entry *cq_entry;
- uint32_t opcode;
-
- cq_entry = (union shomronprm_completion_entry *)cqe;
- cqe_data->owner = MLX_GET ( &cq_entry->normal, owner );
- opcode = MLX_GET ( &cq_entry->normal, opcode );
- #define FLEXBOOT_NODNIC_OPCODE_CQ_SEND 0
- #define FLEXBOOT_NODNIC_OPCODE_CQ_RECV 2
- #define FLEXBOOT_NODNIC_OPCODE_CQ_SEND_ERR 13
- #define FLEXBOOT_NODNIC_OPCODE_CQ_RECV_ERR 14
- cqe_data->is_error =
- ( opcode >= FLEXBOOT_NODNIC_OPCODE_CQ_RECV_ERR);
- if ( cqe_data->is_error ) {
- cqe_data->syndrome = MLX_GET ( &cq_entry->error, syndrome );
- cqe_data->vendor_err_syndrome =
- MLX_GET ( &cq_entry->error, vendor_error_syndrome );
- cqe_data->is_send =
- (opcode == FLEXBOOT_NODNIC_OPCODE_CQ_SEND_ERR);
- } else {
- cqe_data->is_send =
- (opcode == FLEXBOOT_NODNIC_OPCODE_CQ_SEND);
- cqe_data->wqe_counter = MLX_GET ( &cq_entry->normal, wqe_counter );
- cqe_data->byte_cnt = MLX_GET ( &cq_entry->normal, byte_cnt );
-
- }
- if ( cqe_data->is_send == TRUE )
- cqe_data->qpn = MLX_GET ( &cq_entry->normal, qpn );
- else
- cqe_data->qpn = MLX_GET ( &cq_entry->normal, srqn );
-
- return 0;
- }
-
- static mlx_status shomron_cqe_set_owner ( void *cq, unsigned int num_cqes ) {
- unsigned int i = 0;
- union shomronprm_completion_entry *cq_list;
-
- cq_list = (union shomronprm_completion_entry *)cq;
- for ( ; i < num_cqes ; i++ )
- MLX_FILL_1 ( &cq_list[i].normal, 15, owner, 1 );
- return 0;
- }
-
- static mlx_size shomron_get_cqe_size () {
- return sizeof ( union shomronprm_completion_entry );
- }
-
- struct flexboot_nodnic_callbacks shomron_nodnic_callbacks = {
- .get_cqe_size = shomron_get_cqe_size,
- .fill_send_wqe[IB_QPT_ETH] = shomron_fill_eth_send_wqe,
- .fill_completion = shomron_fill_completion,
- .cqe_set_owner = shomron_cqe_set_owner,
- .irq = flexboot_nodnic_eth_irq,
- .tx_uar_send_doorbell_fn = shomron_tx_uar_send_db,
- };
-
- static int shomron_nodnic_is_supported ( struct pci_device *pci ) {
- if ( DEVICE_IS_CIB ( pci->device ) )
- return 0;
-
- return flexboot_nodnic_is_supported ( pci );
- }
- /**************************************************************************/
-
- static int golan_probe ( struct pci_device *pci ) {
- int rc = -ENOTSUP;
-
- DBG ( "%s: start\n", __FUNCTION__ );
-
- if ( ! pci ) {
- DBG ( "%s: PCI is NULL\n", __FUNCTION__ );
- rc = -EINVAL;
- goto probe_done;
- }
-
- if ( shomron_nodnic_is_supported ( pci ) ) {
- DBG ( "%s: Using NODNIC driver\n", __FUNCTION__ );
- rc = flexboot_nodnic_probe ( pci, &shomron_nodnic_callbacks, NULL );
- } else {
- DBG ( "%s: Using normal driver\n", __FUNCTION__ );
- rc = golan_probe_normal ( pci );
- }
-
- probe_done:
- DBG ( "%s: rc = %d\n", __FUNCTION__, rc );
- return rc;
- }
-
- static void golan_remove ( struct pci_device *pci ) {
- DBG ( "%s: start\n", __FUNCTION__ );
-
- if ( ! shomron_nodnic_is_supported ( pci ) ) {
- DBG ( "%s: Using normal driver remove\n", __FUNCTION__ );
- golan_remove_normal ( pci );
- return;
- }
-
- DBG ( "%s: Using NODNIC driver remove\n", __FUNCTION__ );
-
- flexboot_nodnic_remove ( pci );
-
- DBG ( "%s: end\n", __FUNCTION__ );
- }
-
- static struct pci_device_id golan_nics[] = {
- PCI_ROM ( 0x15b3, 0x1011, "ConnectIB", "ConnectIB HCA driver: DevID 4113", 0 ),
- PCI_ROM ( 0x15b3, 0x1013, "ConnectX-4", "ConnectX-4 HCA driver, DevID 4115", 0 ),
- PCI_ROM ( 0x15b3, 0x1015, "ConnectX-4Lx", "ConnectX-4Lx HCA driver, DevID 4117", 0 ),
- PCI_ROM ( 0x15b3, 0x1017, "ConnectX-5", "ConnectX-5 HCA driver, DevID 4119", 0 ),
- PCI_ROM ( 0x15b3, 0x1019, "ConnectX-5EX", "ConnectX-5EX HCA driver, DevID 4121", 0 ),
- PCI_ROM ( 0x15b3, 0x101b, "ConnectX-6", "ConnectX-6 HCA driver, DevID 4123", 0 ),
- PCI_ROM ( 0x15b3, 0x101d, "ConnectX-6DX", "ConnectX-6DX HCA driver, DevID 4125", 0 ),
- PCI_ROM ( 0x15b3, 0xa2d2, "BlueField", "BlueField integrated ConnectX-5 network controller HCA driver, DevID 41682", 0 ),
- };
-
- struct pci_driver golan_driver __pci_driver = {
- .ids = golan_nics,
- .id_count = (sizeof(golan_nics) / sizeof(golan_nics[0])),
- .probe = golan_probe,
- .remove = golan_remove,
- };
|